Change some tests to run with restapi

.github/workflows/pq_pipeline.yml

@@ -131,7 +131,7 @@ jobs:
       (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch')
     name: TaskRunner Dockerized E2E
-    needs: task_runner_straggler_e2e
+    needs: task_runner_resiliency_e2e
     uses: ./.github/workflows/task_runner_dockerized_ws_e2e.yml
     with:
       commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
@@ -173,7 +173,7 @@ jobs:
       (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch')
     name: TaskRunner Verifiable Dataset E2E
-    needs: task_runner_e2e
+    needs: task_runner_connectivity_e2e
     uses: ./.github/workflows/tr_verifiable_dataset_e2e.yml
     with:
       commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

.github/workflows/task_runner_connectivity_e2e.yml

@@ -52,37 +52,36 @@ jobs:
         with:
           test_type: "TLS_Connectivity_gRPC"
 
-# Uncomment once Rest API PR is merged
-  # test_rest_connectivity:
-  #   name: Task Runner Rest connectivity (no-op, 3.11, rest)
-  #   if: |
-  #     (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
-  #     (github.event_name == 'workflow_dispatch') ||
-  #     (github.event.pull_request.draft == false)
-  #   runs-on: ubuntu-22.04
-  #   timeout-minutes: 30
-  #   env:
-  #     MODEL_NAME: 'no-op'
-  #     PYTHON_VERSION: '3.11'
-  #   steps:
-  #     - name: Checkout OpenFL repository
-  #       id: checkout_openfl
-  #       uses: actions/checkout@v4
-  #       with:
-  #         ref: ${{ env.COMMIT_ID }}
+  test_rest_connectivity:
+    name: Task Runner Rest connectivity (no-op, 3.11)
+    if: |
+      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.pull_request.draft == false)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      MODEL_NAME: 'no-op'
+      PYTHON_VERSION: '3.11'
+    steps:
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.COMMIT_ID }}
 
-  #     - name: Pre test run
-  #       uses: ./.github/actions/tr_pre_test_run
-  #       if: ${{ always() }}
+      - name: Pre test run
+        uses: ./.github/actions/tr_pre_test_run
+        if: ${{ always() }}
 
-  #     - name: Run Task Runner rest connectivity test
-  #       id: run_tests
-  #       run: |
-  #         python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_connectivity --model_name ${{ env.MODEL_NAME }} --tr_rest_api
-  #         echo "Task runner end to end test run completed"
+      - name: Run Task Runner rest connectivity test
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_connectivity --model_name ${{ env.MODEL_NAME }} --tr_rest_protocol
+          echo "Task runner end to end test run completed"
 
-  #     - name: Post test run
-  #       uses: ./.github/actions/tr_post_test_run
-  #       if: ${{ always() }}
-  #       with:
-  #         test_type: "TLS_Connectivity_REST"
+      - name: Post test run
+        uses: ./.github/actions/tr_post_test_run
+        if: ${{ always() }}
+        with:
+          test_type: "TLS_Connectivity_REST"

.github/workflows/task_runner_dockerized_ws_e2e.yml

@@ -180,7 +180,7 @@ jobs:
           test_type: "DWS_Without_Client_Auth"
 
   test_memory_logs:
-    name: With Memory Logs
+    name: With Memory Logs REST
     needs: input_selection
     if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs'
     runs-on: ubuntu-22.04
@@ -212,11 +212,11 @@ jobs:
           python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
           -k test_log_memory_usage_dockerized_ws --model_name ${{ env.MODEL_NAME }} \
           --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \
-          --log_memory_usage
+          --log_memory_usage --tr_rest_protocol
           echo "Task runner memory logs test run completed"
 
       - name: Post test run
         uses: ./.github/actions/tr_post_test_run
         if: ${{ always() }}
         with:
-          test_type: "DWS_With_Memory_Logs"
+          test_type: "DWS_With_Memory_Logs_REST"

.github/workflows/task_runner_fed_analytics_e2e.yml

@@ -37,7 +37,7 @@ env:
 
 jobs:
   test_fed_analytics_histogram:
-    name: With TLS (federated_analytics/histogram, 3.11) # DO NOT change this name.
+    name: With REST (federated_analytics/histogram, 3.11) 
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     if: |
@@ -62,7 +62,7 @@ jobs:
       - name: Run Federated Analytics Histogram
         id: run_tests
         run: |
-          python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
+          python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py --tr_rest_protocol \
           -m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
           echo "Federated analytics histogram test run completed"
 
@@ -73,7 +73,7 @@ jobs:
           test_type: "Sepal_Histogram_Analytics"
 
   test_fed_analytics_smokers_health:
-    name: With TLS (federated_analytics/smokers_health, 3.12) # DO NOT change this name.
+    name: With gRPC (federated_analytics/smokers_health, 3.12)
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     if: |

.github/workflows/task_runner_fedeval_e2e.yml

@@ -80,8 +80,8 @@ jobs:
         with:
           test_type: "FedEval_With_TLS"
 
-  test_without_tls:
-    name: Without TLS
+  test_without_tls_rest:
+    name: Without TLS Using Rest Protocol
     if: | # Skip for PR pipeline
       ((github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch')) && (github.workflow != 'OpenFL PR Pipeline')
@@ -115,17 +115,17 @@ jobs:
         run: |
           python -m pytest -s tests/end_to_end/test_suites/tr_with_fedeval_tests.py \
           -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
-          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls
+          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls --tr_rest_protocol
           echo "Task runner end to end test run completed"
 
       - name: Post test run
         uses: ./.github/actions/tr_post_test_run
         if: ${{ always() }}
         with:
-          test_type: "FedEval_Without_TLS"
+          test_type: "FedEval_Without_TLS_REST"
 
-  test_without_client_auth:
-    name: Without Client Auth
+  test_without_client_auth_rest:
+    name: Without ClientAuth Using Rest Protocol
     if: | # Skip for PR pipeline
       ((github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch')) && (github.workflow != 'OpenFL PR Pipeline')
@@ -159,11 +159,11 @@ jobs:
         run: |
           python -m pytest -s tests/end_to_end/test_suites/tr_with_fedeval_tests.py \
           -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
-          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth
+          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth --tr_rest_protocol
           echo "Task runner end to end test run completed"
 
       - name: Post test run
         uses: ./.github/actions/tr_post_test_run
         if: ${{ always() }}
         with:
-          test_type: "FedEval_Without_Client_Auth"
+          test_type: "FedEval_Without_Client_Auth_REST"

.github/workflows/task_runner_resiliency_e2e.yml

@@ -53,50 +53,13 @@ env:
   COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
 
 jobs:
-  input_selection:
-    if: |
-      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event.pull_request.draft == false)
-    name: Input value selection
-    runs-on: ubuntu-22.04
-    outputs:
-      # Output all the variables related to models and python versions to be used in the matrix strategy
-      # for different jobs, however their usage depends on the selected job.
-      selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }}
-      selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }}
-    steps:
-      - name: Job to select input values
-        id: input_selection
-        run: |
-          if [ "${{ env.MODEL_NAME }}" == "all" ]; then
-            echo "models_for_tls=[\"torch/mnist\", \"keras/mnist\"]" >> "$GITHUB_OUTPUT"
-          else
-            echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
-          fi
-          if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then
-            echo "python_for_tls=[\"3.10\", \"3.11\"]" >> "$GITHUB_OUTPUT"
-          else
-            echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
-          fi
-
-  resiliency_in_native:
-    name: With TLS
-    needs: input_selection
+  resiliency_in_native_gRPC:
+    name: Resiliency in gRPC (torch/mnist, 3.10)
     runs-on: ubuntu-22.04
     timeout-minutes: 30
-    strategy:
-      matrix:
-        model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }}
-        python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }}
-        exclude: # Keras does not support Python 3.12
-          - model_name: "keras/mnist"
-            python_version: "3.12"
-      fail-fast: false # do not immediately fail if one of the combinations fail
-
     env:
-      MODEL_NAME: ${{ matrix.model_name }}
-      PYTHON_VERSION: ${{ matrix.python_version }}
+      MODEL_NAME: ${{ inputs.model_name || 'torch/mnist' }}
+      PYTHON_VERSION: ${{ inputs.python_version || '3.10' }}
 
     steps:
       - name: Checkout OpenFL repository
@@ -121,6 +84,38 @@ jobs:
         uses: ./.github/actions/tr_post_test_run
         if: ${{ always() }}
         with:
-          test_type: "Resiliency_Native" 
+          test_type: "Resiliency_Native_gRPC"
+
+  # Uncomment below once Issue is resolved - https://github.com/securefederatedai/openfl/issues/1646
+  # resiliency_in_native_rest:
+  #   name: Resiliency in REST (keras/tensorflow/mnist, 3.11)
+  #   runs-on: ubuntu-22.04
+  #   timeout-minutes: 30
+  #   env:
+  #     MODEL_NAME: ${{ inputs.model_name || 'keras/tensorflow/mnist' }}
+  #     PYTHON_VERSION: ${{ inputs.python_version || '3.11' }}
+
+  #   steps:
+  #     - name: Checkout OpenFL repository
+  #       id: checkout_openfl
+  #       uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{ env.COMMIT_ID }}
+
+  #     - name: Pre test run
+  #       uses: ./.github/actions/tr_pre_test_run
+  #       if: ${{ always() }}
+
+  #     - name: Run Task Runner E2E tests with TLS
+  #       id: run_tests
+  #       run: |
+  #         python -m pytest -s tests/end_to_end/test_suites/tr_resiliency_tests.py \
+  #         -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
+  #         --num_collaborators ${{ env.NUM_COLLABORATORS }} --num_rounds ${{ env.NUM_ROUNDS }} --tr_rest_protocol
+  #         echo "Task runner end to end test run completed"
 
-  # TODO - Add dockerized approach as well once we have GitHub runners with higher configurations.
+  #     - name: Post test run
+  #       uses: ./.github/actions/tr_post_test_run
+  #       if: ${{ always() }}
+  #       with:
+  #         test_type: "Resiliency_Native_REST"

.github/workflows/task_runner_straggler_e2e.yml

@@ -51,16 +51,17 @@ jobs:
         with:
           test_type: "With_TLS_Percentage"
 
+  # Move it to Rest once issue is resolved https://github.com/securefederatedai/openfl/issues/1646
   test_straggler_cutoff:
-    name: Cutoff Policy (torch/mnist_straggler_check, 3.10)
+    name: Cutoff Policy (torch/mnist_straggler_check, 3.11)
     if: |
       (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     env:
       MODEL_NAME: 'torch/mnist_straggler_check'
-      PYTHON_VERSION: '3.10'
+      PYTHON_VERSION: '3.11'
     steps:
       - name: Checkout OpenFL repository
         id: checkout_openfl

.github/workflows/windows.yml

@@ -54,6 +54,24 @@ jobs:
         run: |
           python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
 
+  torch_mnist_rest: # from taskrunner.yml - torch/mnist
+    if: github.event.pull_request.draft == false
+    runs-on: windows-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+      - name: Test TaskRunner API
+        run: |
+          python -m tests.github.test_hello_federation --template torch/mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --transport-protocol rest --save-model output_model
+
   torch_mnist_eden_compression: # from taskrunner_eden_pipeline.yml - torch/mnist_eden_compression
     if: github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression')
     runs-on: windows-latest

tests/end_to_end/conftest.py

@@ -31,7 +31,7 @@ def pytest_addoption(parser):
     parser.addoption("--num_rounds")
     parser.addoption("--model_name")
     parser.addoption("--workflow_backend")
-    parser.addoption("--tr_rest_api", action="store_true")
+    parser.addoption("--tr_rest_protocol", action="store_true")
     parser.addoption("--disable_client_auth", action="store_true")
     parser.addoption("--disable_tls", action="store_true")
     parser.addoption("--log_memory_usage", action="store_true")
@@ -54,7 +54,7 @@ def pytest_configure(config):
     config.use_tls = not args.disable_tls
     config.log_memory_usage = args.log_memory_usage
     config.secure_agg = args.secure_agg
-    config.tr_rest_api = args.tr_rest_api
+    config.tr_rest_protocol = args.tr_rest_protocol
     config.workflow_backend = args.workflow_backend
     config.results_dir = config.getini("results_dir")
 

tests/end_to_end/models/aggregator.py

@@ -21,20 +21,22 @@ class Aggregator():
     2. Starting the aggregator
     """
 
-    def __init__(self, agg_domain_name, workspace_path, eval_scope=False, container_id=None):
+    def __init__(self, agg_domain_name, workspace_path, transport_protocol, eval_scope=False, container_id=None):
         """
         Initialize the Aggregator class
         Args:
             agg_domain_name (str): Aggregator domain name
             workspace_path (str): Workspace path
             container_id (str): Container ID
             eval_scope (bool, optional): Scope of aggregator is evaluation. Default is False.
+            transport_protocol (str): Transport protocol (default: "gRPC")
         """
         self.name = "aggregator"
         self.agg_domain_name = agg_domain_name
         self.workspace_path = workspace_path
         self.eval_scope = eval_scope
         self.container_id = container_id
+        self.transport_protocol = transport_protocol
         self.tensor_db_file = os.path.join(self.workspace_path, "local_state", "tensor.db")
         self.res_file = None # Result file to track the logs
         self.start_process = None # Process associated with the aggregator start command
@@ -87,7 +89,7 @@ def start(self):
                 cmd=command,
                 work_dir=self.workspace_path,
                 redirect_to_file=bg_file,
-                check_sleep=60,
+                check_sleep=30,
                 env=env
             )