.gitlab-ci.yml

variables:
  MODE: debug # run aistore and tests with debug asserts enabled
  CI_IMAGE: aistorage/gitlab-ci:latest
  RUNNER_TAG: "ais-ci"
  SCRIPTS_DIR: "./scripts"
  NUM_TARGET:
    value: "6"
    description: "Number of targets."
  NUM_PROXY:
    value: "6"
    description: "Number of proxies."
  FS_CNT:
    value: "4"
    description: "Number of mountpaths (disks) in each target."
  CLOUD_BCKS:
    value: "aws://ais-blossom gs://ais-ci-kube oc://ais-ci-long" # NOTE: additionally used: aws://ais-cicd and aws://ais-ci-python
    description: "Set of buckets used when running cloud tests."
  GOOGLE_APPLICATION_CREDENTIALS:
    value: "$GCP_JSON_FILE"
    description: "GitLab file variable containing credentials for GCP"
  ORACLE_PRIVATE_KEY:
    value: "$OCI_PRIVATE_KEY_FILE"
    description: "GitLab file variable containing private key for OCI"

image: $CI_IMAGE

stages:
  - build
  - test-short
  - test-long


# Rules

.changes_rules:
  rules:
    - &etl_changes_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - "**/etl/**/*"
        - "**/*etl*.py"
        - "**/*etl*.go"
        - 'api/apc/*'
        - 'cmn/k8s/*'
        - 'core/lom*'
        - 'core/ldp.go'
        - 'core/xaction.go'
        - 'xact/xreg/*'
        - 'xact/xs/tcb.go'
        - 'xact/xs/tcobjs.go'
    - &authn_changes_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - "**/authn/**/*"
        - '**/*authn*.go'
        - '**/*authn*.py'
        - 'api/apc/*'
    - &auth_labels_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*authn?.*/'
    - &python_req_changes_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - 'python/aistore/common_requirements'
    - &pytorch_label_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*pytorch.*/'
    - &pytorch_changes_rule
      if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - 'python/aistore/pytorch/**/*'
        - 'python/tests/unit/pytorch/**/*'
        - 'python/tests/integration/pytorch/**/*'

.default_rules: &default_rules
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

.default_test_rules: &default_test_rules
  rules:
    - !reference [.default_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true

.default_run_rules: &default_run_rules
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'

.skip_scheduled_rules: &skip_scheduled_rules
  rules:
    - !reference [.default_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
      allow_failure: true


# Templates

.gather_logs_template: &gather_logs_def
  after_script:
    - make kill # To make sure that nodes flushed the logs.
    - mkdir -p $CI_PROJECT_DIR/logs $CI_PROJECT_DIR/logs/remote
    - find /tmp/ais -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/ \; 2>/dev/null || true
    - find /tmp/ais_next -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/remote/ \; 2>/dev/null || true
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.test_short_template: &test_short_def
  stage: test-short
  tags:
    - $RUNNER_TAG
  timeout: 53m
  rules:
  <<: *gather_logs_def
  <<: *default_test_rules

.test_short_skip_scheduled_template: &test_short_skip_scheduled_def
  stage: test-short
  tags:
    - $RUNNER_TAG
  timeout: 53m
  <<: *gather_logs_def
  <<: *skip_scheduled_rules

.test_short_optional_template: &test_short_optional_def
  stage: test-short
  tags:
    - $RUNNER_TAG
  timeout: 53m
  rules:
    - !reference [.default_test_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_template: &test_long_def
  stage: test-long
  tags:
    - $RUNNER_TAG
  timeout: 6h
  rules:
    - !reference [.default_test_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_skip_scheduled_template: &test_long_skip_scheduled_def
  stage: test-long
  tags:
    - $RUNNER_TAG
  timeout: 6h
  <<: *skip_scheduled_rules
  <<: *gather_logs_def


# Stages

build:linux:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - !reference [.default_rules, rules]
    - !reference [.default_run_rules, rules]
  script:
    - MODE="" make node # Build node without backends in production mode.
    - MODE="debug" make node # Build node without backends in debug mode.
    - AIS_BACKEND_PROVIDERS="aws azure gcp oci" MODE="" make node # Build with all Cloud backends (production mode).
    - AIS_BACKEND_PROVIDERS="aws azure gcp oci ht" MODE="debug" make node # Build with all backends (debug mode).
    - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile.
    - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default).
    - TAGS="oteltracing" make node # Build with oteltracing (disabled by default).
    - make authn
    - make cli
    - make aisloader

security:sonarqube:
  stage: build
  tags:
    - $RUNNER_TAG
    - gp-vm
  image:
    name: sonarsource/sonar-scanner-cli:latest
    entrypoint: [""]
  variables:
    SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"
    GIT_DEPTH: "0"
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .sonar/cache
  allow_failure: true
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
    - !reference [.default_run_rules, rules]
  script:
    - sonar-scanner

build:k8s:
  stage: build
  tags:
    - $RUNNER_TAG
  variables:
    # Assuming runners are using overlayfs, we must use vfs to avoid layered overlayfs
    STORAGE_DRIVER: vfs
  before_script:
    - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_TOKEN $CI_REGISTRY
    - buildah login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_TOKEN docker.io
  script:
    - make -C deploy/prod/k8s/aisnode_container ci
    - make -C deploy/prod/k8s/aisinit_container ci
  rules:
    - !reference [.test_k8s_short_template, rules]
    - *python_req_changes_rule

lint:fmt:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - !reference [ .default_rules, rules ]
    - !reference [ .default_run_rules, rules ]
  script:
    - make fmt-check
    - make spell-check

lint:go:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - !reference [.default_rules, rules]
    - !reference [.default_run_rules, rules]
  script:
    - golangci-lint config verify
    - make lint

lint:python:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - !reference [.default_rules, rules]
    - !reference [.default_run_rules, rules]
  script:
    - make lint-python

lint:tags:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - !reference [.default_rules, rules]
    - !reference [.default_run_rules, rules]
  script:
    - TAGS="dsort nethttp debug oteltracing oci aws gcp azure" make lint-scoped

build:gendocs:
  stage: build
  tags:
    - $RUNNER_TAG
  timeout: 10m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*website.*/'
  variables:
    GENERATOR_IMG: docker.io/openapitools/openapi-generator-cli:v7.14.0
  script:
    - go run ./tools/gendocs
    - podman run --rm -v "${PWD}:/work:z" "${GENERATOR_IMG}" generate -i /work/.docs/openapi.yaml -g markdown -o /work/docs-generated --template-dir /work/markdown-template
    - ./scripts/website-preprocess.sh

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:
  <<: *test_short_def
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/'
      when: manual
      allow_failure: true
    - !reference [.default_run_rules, rules]
  variables:
    BUCKET: "ais://test"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short
    - make test-tracing-unit

test:short:aisloader:
  <<: *test_short_def
  timeout: 20m
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/'
      when: manual
      allow_failure: true
    - !reference [.default_run_rules, rules]
  script:
    - make aisloader
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - FLAGS="--duration=2m" make test-aisloader

test:short:python-changelog-check:
  <<: *test_short_def
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-python-changelog-check.*/'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - python/aistore/sdk/**/*
        - python/aistore/version.py
  script:
    - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
    - CHANGED_FILES=$(git diff --name-only "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"...HEAD)
    - echo "Changed files are:"
    - echo "$CHANGED_FILES"
    - echo "$CHANGED_FILES" | grep -qx 'python/CHANGELOG.md' || exit 1

test:short:python-unit:
  <<: *test_short_def
  rules:
    - !reference [.test_short_template, rules]
    - !reference [.default_run_rules, rules]
  variables:
    PIP: "uv pip"
  script:
    - cd python
    - |
      set -e
      for ver in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
        uv venv --python "$ver" .venv
        source .venv/bin/activate
        make python_sdk_unit_tests
        make python_unit_tests_scenario_gen
        make python_botocore_unit_tests
        make python_mcp_unit_tests
        deactivate || true
        rm -rf .venv
      done

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:python-integration:
  <<: *test_short_def
  rules:
    - !reference [.test_short_template, rules]
    - !reference [.default_run_rules, rules]
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
    BUCKET: "aws://ais-ci-python"
    PIP: "uv pip"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
    - cd python
    - |
      set -e
      for ver in 3.8 3.9 3.14; do
        uv venv --python "$ver" .venv
        source .venv/bin/activate
        make python_botocore_integration_tests
        # TODO: For now, only run full SDK integration on 3.14
        if [ "$ver" = "3.14" ]; then make python_sdk_integration_tests; fi
        deactivate || true
        rm -rf .venv
      done

test:short:pytorch:
  <<: *test_short_optional_def
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
    BUCKET: "aws://ais-ci-python"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
    - cd python
    - make python_pytorch_tests
  rules:
    - !reference [.test_short_template, rules]
    - *pytorch_label_rule
    - *pytorch_changes_rule
    - !reference [.test_short_optional_template, rules]

test:short:python-authn:
  stage: test-short
  tags:
    - $RUNNER_TAG
  timeout: 20m
  <<: *gather_logs_def
  variables:
    AIS_AUTHN_DEPLOY: "true"
    AIS_AUTHN_ENABLED: "true"
    AIS_AUTHN_SU_NAME: "admin"
    AIS_AUTHN_SU_PASS: "admin"
    AIS_AUTHN_SECRET_KEY: "gitlabCISigningKey"
    AIS_AUTHN_URL: "http://localhost:52001"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - cd python/
    - make python_authn_tests
  rules:
    - !reference [.test_short_template, rules]
    - *auth_labels_rule
    - *python_req_changes_rule
    - *authn_changes_rule

# Runs cluster with 1 proxy and 1 target (with 6 mountpaths).
test:short:minimal:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/'
      when: manual
      allow_failure: true
    - !reference [.default_run_rules, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all
    - NUM_PROXY=1 NUM_TARGET=1 make test-short

test:short:chunks:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
    NUM_CHUNKS: "4"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*chunks.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short

test:short:sign_hmac:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
    SIGN_HMAC: "true"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*sign_hmac.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short

test:short:rand_ns:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
    RAND_NS: "true"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*sign_rand_ns.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short

test:short:s3-compat:
  <<: *test_short_def
  tags:
    - $RUNNER_TAG
  variables:
    # Run only s3 compat tests we know should pass
    S3_COMPAT_RUN_ALL:
      value: "False"
    # Fail on first failure
    S3_COMPAT_STRICT:
      value: "True"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - ais config cluster features S3-API-via-Root
    - ais config cluster checksum.type=md5
    - cd python
    - make python_s3_compat_test
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS !~ /.*s3-compat.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never

test:short:authn:
  <<: *test_short_optional_def
  variables:
    AIS_AUTHN_DEPLOY: "true"
    AIS_AUTHN_ENABLED: "true"
    AIS_AUTHN_SU_NAME: "admin"
    AIS_AUTHN_SU_PASS: "admin"
    AIS_AUTHN_SECRET_KEY: "gitlabCISigningKey"
    AIS_AUTHN_URL: "http://localhost:52001"
    BUCKET: "ais://test"
    RE: "TestAuth"
    TEST_RACE: "true"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - ais auth login $AIS_AUTHN_SU_NAME -p $AIS_AUTHN_SU_PASS
    - make test-run
    - ais auth logout
  rules:
      - !reference [ .default_test_rules, rules ]
      - *auth_labels_rule
      - *authn_changes_rule
      - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
      - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        when: manual
        allow_failure: true

test:short:https:
  <<: *test_short_def
  variables:
    AIS_USE_HTTPS: "true"
    AIS_SKIP_VERIFY_CRT: "true"
    AIS_SERVER_CRT: "$CI_PROJECT_DIR/localhost.crt"
    AIS_SERVER_KEY: "$CI_PROJECT_DIR/localhost.key"
    AIS_ENDPOINT: "https://localhost:8080"
    BUCKET: "ais://ais-ci"
  script:
    - openssl req -x509 -out $AIS_SERVER_CRT -keyout $AIS_SERVER_KEY -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <( printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth")
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https
    - make test-short
  rules:
    - !reference [.test_short_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*https.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS !~ /.*https.*/'
      when: manual
      allow_failure: true

test:short:ipv6:
  <<: *test_short_def
  variables:
    AIS_USE_IPv6: "true"
    BUCKET: "ais://ais-ci"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*ipv6.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-short

test:short:aws:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "aws://ais-cicd"
  timeout: 70m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*aws.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws
    - make test-short

test:short:gcp:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "gs://ais-ci"
  timeout: 70m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*gcp.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-short

test:short:azure:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "az://ais-ci"
  timeout: 70m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*azure.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure
    - make test-short

test:short:oci:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "oc://ais-ci"
  timeout: 2h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*oci.*/'
    - !reference [.test_short_skip_scheduled_template, rules]
  script:
    - . ais/test/oci/mp_tuning.env && ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --oci
    - . ais/test/oci/mp_tuning.env && make test-short

test:long:
  <<: *test_long_skip_scheduled_def
  variables:
    BUCKET: "ais://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --ht --deployment all
    - make test-long

test:long:python-extended:
  <<: *test_long_def
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
    BUCKET: "aws://ais-ci-python"
    PIP: "uv pip"
  timeout: 35m
  before_script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
  script:
    - cd python
    - uv venv --python 3.14 .venv
    - source .venv/bin/activate
    - make python_sdk_extended_tests
    - deactivate || true
    - rm -rf .venv

test:long:aws:
  <<: *test_long_def
  variables:
    BUCKET: "aws://ais-cicd"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*aws.*/'
    - !reference [.test_long_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws --ht --deployment all
    - make test-long

test:long:gcp:
  <<: *test_long_def
  variables:
    BUCKET: "gs://ais-ci"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*gcp.*/'
    - !reference [.test_long_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --ht --deployment all
    - make test-long

test:long:azure:
  <<: *test_long_def
  variables:
    BUCKET: "az://ais-ci"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*azure.*/'
    - !reference [.test_long_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure --ht --deployment all
    - make test-long

test:long:oci:
  <<: *test_long_def
  variables:
    BUCKET: "oc://ais-ci"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*oci.*/'
    - !reference [.test_long_skip_scheduled_template, rules]
  script:
    - . ais/test/oci/mp_tuning.env && ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --oci --ht --deployment all
    - . ais/test/oci/mp_tuning.env && make test-long

test:long:aisloader:
  stage: test-long
  tags:
    - $RUNNER_TAG
  timeout: 15m
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - FLAGS="--duration=5m" make test-aisloader
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

test:long:pyaisloader:
  stage: test-long
  tags:
    - $RUNNER_TAG
  timeout: 15m
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - make -C python/pyaisloader ci-long
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

test:long:ishard:
  stage: test-long
  tags:
    - $RUNNER_TAG
  timeout: 1h
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - cd ./cmd/ishard/ishard; go test -v -timeout=10h -run=^TestIshard
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

test:long:chunks:
  <<: *test_long_def
  variables:
    BUCKET: "ais://test"
    NUM_CHUNKS: "4"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: manual
      allow_failure: true
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*chunks.*/'
    - !reference [.test_long_skip_scheduled_template, rules]
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-long

#
# Kubernetes stages
#

.test_k8s:
  tags:
    - $RUNNER_TAG
  variables:
    BUCKET: "gs://ais-ci-kube"
    TESTS_DIR: "ais/test"
    AISNODE_IMAGE: "$CI_REGISTRY_IMAGE/aisnode"
    AISINIT_IMAGE: "$CI_REGISTRY_IMAGE/ais-init"
    FS_CNT: 6
    RE: "TestETL|TestConfig|TestMountpath|TestBlobDownload"
    AIS_BACKEND_PROVIDERS: "gcp aws oci"
  rules:
    - !reference [.default_test_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
  before_script:
    - cd deploy/dev/k8s
    - ./utils/ci/generate_kind_config.sh > kind_config.yml
    - kind create cluster --config=kind_config.yml --wait=5m
    - kubectl cluster-info  # wait for k8s
    - make gitlab-ci
    - source ./export_endpoint.sh
    - cd ../../..
    - make cli
  after_script:
    - mkdir $CI_PROJECT_DIR/logs
    - cp -r /ais/log $CI_PROJECT_DIR/logs
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.test_k8s_short_template:
  stage: test-short
  extends: .test_k8s
  variables:
    NUM_PROXY: 1
    NUM_TARGET: 3
  rules:
    - !reference [.test_k8s, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*etl.*/'
    - *etl_changes_rule

.test_k8s_long_template:
  stage: test-long
  extends: .test_k8s
  variables:
    NUM_PROXY: 1
    NUM_TARGET: 3
  rules:
    - !reference [.default_rules, rules]
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
      allow_failure: true

test:short:k8s:
  extends: .test_k8s_short_template
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/'
      when: manual
      allow_failure: true
    - !reference [.test_k8s_short_template, rules]
  variables:
    RE: "TestETL|TestConfig|TestMountpath|TestBlobDownload"
  script:
    - make test-short

test:short:python-etl:
  extends: .test_k8s_short_template
  rules:
    - !reference [.test_k8s_short_template, rules]
    - *python_req_changes_rule
  retry:
    max: 2
    when:
      - unknown_failure
      - api_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - job_execution_timeout
  variables:
    PIP: "uv pip"
  script:
    - cd python
    - uv venv --python 3.13 .venv
    - source .venv/bin/activate
    - make python_etl_tests
    - deactivate || true
    - rm -rf .venv

#  e.g. RE: "ETLBucket|ETLFQN" (or any other regex to select tests)
test:short:assorted:k8s:
  extends: .test_k8s_short_template
  timeout: 35m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  variables:
    RE: "ETLAnyToAny|ETLMultiObj"
  script:
    - make test-run

test:long:k8s:
  extends: .test_k8s_long_template
  timeout: 6h
  rules:
    - !reference [.test_k8s_long_template, rules]
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - make test-run

test:long:k8s:single-target:
  extends: .test_k8s_long_template
  timeout: 6h
  variables:
    NUM_TARGET: 1
  script:
    - make test-run

test:long:k8s:aisloader:
  extends: .test_k8s_long_template
  timeout: 15m
  script:
    - sleep 10 # Give some time for the cluster to stabilize.
    - make aisloader
    - BUCKET="ais://test" FLAGS="--duration=2m --etl" make test-aisloader

test:long:k8s:pyaisloader:
  extends: .test_k8s_long_template
  timeout: 15m
  script:
    - sleep 10 # Give some time for the cluster to stabilize.
    - make -C python/pyaisloader ci-etl

test:long:k8s:all:
  extends: .test_k8s_long_template
  timeout: 6h
  variables:
    AIS_BACKEND_PROVIDERS: "gcp aws oci"
  before_script:
    - make aisloader
    - cd deploy/dev/k8s
    - ./utils/ci/generate_kind_config.sh > kind_config.yml
    - kind create cluster --config=kind_config.yml
    - kubectl cluster-info  # wait for k8s
    - make gitlab-ci
    - source ./export_endpoint.sh
    - cd ../../..
    # Make sure that metrics collection is enabled.
    - git clone https://github.com/prometheus-operator/kube-prometheus.git
    - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus
  script:
    - |
      echo "----- RUNNING K8S TESTS -----"
      BUCKET="aws://ais-blossom" RE="TestETL|TestConfig|TestMountpath|TestBlobDownload" make test-run
      exit_code=$?
      result=$((result + exit_code))
      echo "----- K8S TESTS FINISHED WITH: ${exit_code} -----"
    - |
      for bucket in ${CLOUD_BCKS}; do
        echo "----- RUNNING LONG TESTS WITH: ${bucket} -----"
        BUCKET=${bucket} make test-long && make test-aisloader
        exit_code=$?
        result=$((result + exit_code))
        echo "----- LONG TESTS FINISHED WITH: ${exit_code} -----"
      done