Updating code to work with current versions of pydantic and prefect

.prefectignore

@@ -0,0 +1,18 @@
+.git
+.github
+.gitignore
+.idea
+.*_cache
+__pycache__
+.pre-commit-config.yaml
+docker/
+docs/
+site/
+tests/
+example-config.yml
+example-openapi.yml
+LICENSE
+mkdocs.yml
+poetry.lock
+pyproject.toml
+README.md

dev-log-config.yaml

@@ -0,0 +1,65 @@
+# uvicorn logging configuration suitable for development
+#
+# This mostly copies the default uvicorn config and then adds an extra logger
+# that logs messages of the `pygeoapi` and `prefect` loggers
+
+version: 1
+disable_existing_loggers: false
+
+formatters:
+
+  default:
+    (): uvicorn.logging.DefaultFormatter
+    fmt: "%(levelprefix)s %(message)s"
+    use_colors: null
+
+  access:
+    (): uvicorn.logging.AccessFormatter
+    fmt: '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s'
+
+handlers:
+
+  default:
+    formatter: default
+    class: logging.StreamHandler
+    stream: ext://sys.stderr
+
+  access:
+    formatter: access
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+
+loggers:
+
+  uvicorn:
+    handlers:
+      - default
+    level: INFO
+    propagate: false
+
+  uvicorn.error:
+    level: INFO
+
+  uvicorn.access:
+    handlers:
+      - access
+    level: INFO
+    propagate: False
+
+  app:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+
+  pygeoapi:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+
+  pygeoapi_prefect:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false

docker/Dockerfile

@@ -0,0 +1,68 @@
+FROM python:3.10-slim-bullseye
+
+# Install security updates and system dependencies, then clean up
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get --yes upgrade && \
+    # these are our own dependencies and utilities
+    # if you need to add more, please sort them in alphabetical order
+    apt-get install --yes --no-install-recommends \
+      curl \
+      net-tools \
+      procps \
+      tini && \
+    apt-get --yes clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# download poetry
+RUN curl --silent --show-error --location \
+    https://install.python-poetry.org > /opt/install-poetry.py
+
+# Create a normal non-root user so that we can use it to run
+RUN useradd --create-home appuser
+
+# Compile python stuff to bytecode to improve startup times
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10)"
+
+USER appuser
+
+#RUN mkdir /home/appuser/app  && \
+#    mkdir /home/appuser/data && \
+#    python opt/install-poetry.py --yes --version 1.4.2
+
+
+RUN mkdir --parents /home/appuser/dev/geobeyond/pygeoapi-prefect  && \
+    python opt/install-poetry.py --yes --version 1.4.2
+
+ENV PATH="$PATH:/home/appuser/.local/bin" \
+    # This allows us to get traces whenever some C code segfaults
+    PYTHONFAULTHANDLER=1
+
+# Copy over pygeoapi source
+WORKDIR /home/appuser/dev
+COPY --chown=appuser:appuser ../../pygeoapi .
+
+# Only copy the dependencies for now and install them
+# WORKDIR /home/appuser/app
+WORKDIR /home/appuser/dev/geobeyond/pygeoapi-prefect
+COPY --chown=appuser:appuser pyproject.toml poetry.lock ./
+RUN poetry install --no-root --only main
+
+EXPOSE 5000
+
+# Now install our code
+COPY --chown=appuser:appuser . .
+RUN poetry install --only main
+
+# Write git commit identifier into the image
+ARG GIT_COMMIT
+ENV GIT_COMMIT=$GIT_COMMIT
+RUN echo $GIT_COMMIT > /home/appuser/git-commit.txt
+
+# Compile python stuff to bytecode to improve startup times
+RUN poetry run python -c "import compileall; compileall.compile_path(maxlevels=10)"
+
+# use tini as the init process
+ENTRYPOINT ["tini", "-g", "--", "poetry", "run", "pygeoapi"]
+
+CMD ["serve"]

docker/compose.yaml

@@ -0,0 +1,40 @@
+# Base docker-compose file
+#
+# This file has the main compose configuration for running the project. Specialize it into suitable `dev` and `ci`
+# stacks in the `docker-compose.dev.yml` and `docker-compose.ci` files, as needed
+
+name: pygeoapi-prefect
+
+services:
+
+  prefect-server:
+    image: prefecthq/prefect:2-python3.10
+    ports:
+      - target: 4200
+        published: 44200
+    environment:
+      PREFECT_UI_API_URL: "http://localhost:44200/api"
+    command: ["prefect", "server", "start", "--host", "0.0.0.0"]
+
+#  prefect-agent:
+#    image: prefecthq/prefect:2-python3.10
+#    environment:
+#      PREFECT_API_URL: "http://prefect-server:4200/api"
+#    command: ["prefect", "agent", "start", "-q", "pygeoapi"]
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - target: 9000
+        published: 9000
+      - target: 9090
+        published: 9090
+    environment:
+      MINIO_ROOT_USER: tester
+      MINIO_ROOT_PASSWORD: 12345678
+    volumes:
+      - minio-data:/data
+    command: ["server", "/data", "--console-address", ":9090"]
+
+volumes:
+  minio-data:

docs/development.md

@@ -28,9 +28,28 @@ poetry run prefect agent start --work-queue pygeoapi
 Now stand up pygeoapi with the provided config files:
 
 ```shell
-PYGEOAPI_CONFIG=example-config.yml PYGEOAPI_OPENAPI=example-openapi.yml poetry run pygeoapi serve
+export PYGEOAPI_INSTALL_DIR=${HOME}/dev/pygeoapi
+export PYGEOAPI_CONFIG=example-config.yml
+export PYGEOAPI_OPENAPI=example-openapi.yml
+
+poetry run uvicorn \
+       --reload \
+       --reload-dir=${PYGEOAPI_INSTALL_DIR} \
+       --reload-dir=$(pwd) \
+       --reload-include='*.py' \
+       --reload-include='*.yml' \
+       --host=0.0.0.0 \
+       --port=5000 \
+       --log-level=debug \
+       --log-config=dev-log-config.yaml \
+       pygeoapi.starlette_app:APP
 ```
 
+!!! NOTE
+
+    It is preferable to call `uvicorn` directly from the CLI over using `pygeoapi serve` as
+    that will allow setting custom uvicorn flags, like the log config
+
 If you need to regenerate the openapi description file, run:
 
 ```shell

docs/index.md

@@ -14,7 +14,7 @@ A [pygeoapi] job/process manager that enables running pygeoapi jobs as [prefect]
 
 ## Quickstart
 
-Follow the [user guide](user-guide/user-guide.md) for installation and initial usage instructions.
+Follow the [user guide](user-guide/installation.md) for installation and initial usage instructions.
 
 
 ## License

docs/user-guide/complete-example.md

@@ -9,31 +9,56 @@ We will configure pygeoapi to use minIO for both:
 - Storing the deployed flow code
 
 
-Let's start by standing up a local docker container with minIO:
+Let's start by spinning up the services defined in the `docker/compose.yaml` stack file,
+which are:
+
+- prefect-server
+- minio
+
+!!! NOTE
+
+    When running prefect from the supplied `docker/compose.yaml` docker
+    Compose stack, prefect server will be running locally on port 44200, which is not
+    The standard port. In this case it may be easier to create a prefect profile and use
+    That:
+
+      ```sh
+      # you must have prefect installed locally
+      prefect profile create pygeoapi-prefect-dev
+      prefect profile use pygeoapi-prefect-dev
+      prefect config set PREFECT_API_URL=http://0.0.0.0:44200
+      ```
+
+Additionally you will need to run:
+
+-  pygeoapi server
+
+   ```sh
+   export PYGEOAPI_CONFIG=example-config.yml
+   export PYGEOAPI_OPENAPI=example-openapi.yml
+
+   pygeoapi serve
+   ```
+
+-  prefect agent
+
+   ```sh
+   prefect agent start --work-queue pygeoapi
+   ```
 
-```shell
-mkdir --parents ~/minio/data
-
-docker run \
-    --publish 9000:9000 \
-    --publish 9090:9090 \
-    --name minio \
-    --volume ~/minio/data:/data \
-    --env "MINIO_ROOT_USER=tester" \
-    --env "MINIO_ROOT_PASSWORD=12345678" \
-    quay.io/minio/minio \
-    server /data --console-address ":9090"
-```
 
 Login to the minIO dashboard at http://127.0.0.1:9090 then go ahead and create a bucket named `pygeoapi-test`
 
 Now you need to create a prefect storage block that references this bucket. Either use the custom
 CLI command shipped by pygeoapi-prefect or create the block using prefect UI.
 
-Using the pygeoapi-prefect CLI command:
+Using the custom CLI command:
 
 ```shell
-pygeoapi plugins prefect create-storage-block \
+export PYGEOAPI_CONFIG=example-config.yml
+export PYGEOAPI_OPENAPI=example-openapi.yml
+
+pygeoapi plugins prefect create-remote-storage-block \
     test-sb1 \
     s3://pygeoapi-test \
     http://localhost:9000 \
@@ -60,6 +85,24 @@ Using the Prefect UI you would create a block of type _Remote File System_:
 ## Implement process code and configure pygeoapi
 
 For this example we are using the example flow that is shipped with pygeoapi-prefect.
+It uses the following configuration:
+
+```yaml
+# pygeoapi configuration file: example-config.yml
+
+resources:
+  simple-flow:
+    type: process
+    processor:
+      name: pygeoapi_prefect.examples.simple_prefect.SimpleFlowProcessor
+      prefect:
+        result_storage: remote-file-system/test-sb1-results
+        deployment:
+          name: minio
+          queue: pygeoapi
+          storage_block: remote-file-system/test-sb1
+          storage_sub_path: simple-flow-flow
+```
 
 
 ## Deploy process
@@ -72,17 +115,28 @@ export PYGEOAPI_CONFIG=example-config.yml
 poetry run pygeoapi plugins prefect deploy-process hi-prefect-world
 ```
 
-This results in prefect creating a deployment named `hi-prefect-world/test`, and since we are specifying a storage
+This results in prefect creating a deployment named `hi-prefect-world/minio`, and since we are specifying a storage
 block, prefect also uploads the flow code onto the storage (which is the minIO bucket created previously).
 
 You can verify this step by:
 
-- Checking the presence of a `hi-prefect-world-flow` entry inside the previously created minIO bucket. By running
+- Checking the presence of a `simple-flow-flow` entry inside the previously created minIO bucket. By running
   the `deploy-process` command you instructed prefect to upload the flow code to the minIO storage.
-- Checking the existence of a `hi-prefect-world/minio` deployment, as reported in the prefect UI
+- Checking the existence of a deployment named `minio` and that has `simple-flow` as its flow name by using
+  the prefect UI
+
+
+## Execute process via pygeoapi
+
+With our deployment ready, we can now run it via several mechanisms:
+
+- By using pygeoapi's OAProc API
+- Using the prefect CLI
+- Using the prefect GUI
+
 
+### Executing process via pygeoapi
 
-## 3. Execute process via pygeoapi
 
 ```shell
 http -v localhost:5000/processes/hi-prefect-world/execution inputs:='{"name": "Frankie Four-fingers"}'

docs/user-guide/defining-processes.md

@@ -39,7 +39,7 @@ As such, in order to use prefect-powered processes you will need to:
 
 !!! note
 
-    We recommend getting having the [prefect docs](https://docs.prefect.io/latest/) at hand when defining new flows.
+    We recommend having the [prefect docs](https://docs.prefect.io/latest/) at hand when defining new flows.
 
 
 pygeoapi-prefect is able to use regular prefect flows. However, they must meet the following requirements:

docs/user-guide/deployments.md

@@ -12,7 +12,7 @@ A prefect deployment thus stores metadata about the storage location of a flow's
 
 Deployments are inspected by the prefect agent when it is time to run a scheduled flow. these are used by the agent to:
 
-- Retrieve flow code from it storage location and make it available for execution
+- Retrieve flow code from its storage location and make it available for execution
 - Set up a suitable execution environment for the flow run. For example: spin up a docker container and install Python
   dependencies