Merge branch 'main' into job_parent_hierarchy_backfills

Author: collado-mike

.circleci/config.yml

@@ -29,6 +29,7 @@ jobs:
             - v1-api-{{ .Branch }}
       - run: ./.circleci/get-jdk17.sh
       - run: echo "checks.disable=true" > ~/.testcontainers.properties
+      - run: ./gradlew --no-daemon --stacktrace api:javadoc
       - run: ./gradlew --no-daemon --stacktrace api:build
       - run: ./gradlew --no-daemon api:jacocoTestReport
       - run: bash <(curl -s https://codecov.io/bash)
@@ -95,6 +96,7 @@ jobs:
             - v1-client-java-{{ .Branch }}-{{ .Revision }}
             - v1-client-java-{{ .Branch }}
       - run: ./.circleci/get-jdk17.sh
+      - run: ./gradlew --no-daemon --stacktrace clients:java:javadoc
       - run: ./gradlew --no-daemon --stacktrace clients:java:build
       - run: ./gradlew --no-daemon clients:java:jacocoTestReport
       - run: bash <(curl -s https://codecov.io/bash)
@@ -132,7 +134,7 @@ jobs:
             - ./dist/*.whl
             - ./dist/*.tar.gz
 
-  lint-docs-api:
+  lint-spec-api:
     working_directory: ~/marquez
     docker:
       - image: cimg/node:12.22.7
@@ -187,7 +189,7 @@ workflows:
       - build-client-java
       - unit-test-web
       - unit-test-client-python
-      - lint-docs-api
+      - lint-spec-api
   release:
     jobs:
       - build-client-python:

CHANGELOG.md

@@ -1,16 +1,27 @@
 # Changelog
 
-## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.21.0...HEAD)
+## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.22.0...HEAD)
+
+## [0.22.0](https://github.com/MarquezProject/marquez/compare/0.21.0...0.22.0) - 2022-05-16
 
 ### Added
 
-* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
-* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [@wslulciuc](https://github.com/wslulciuc)
-* Add support for job renaming/redirection via symlink [@collado-mike](https://github.com/collado-mike)
+* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [#1847](https://github.com/MarquezProject/marquez/pull/1847)[@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
+* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [#1945](https://github.com/MarquezProject/marquez/pull/1945) [@wslulciuc](https://github.com/wslulciuc)
+* Add support for job renaming/redirection via symlink [#1947](https://github.com/MarquezProject/marquez/pull/1947) [@collado-mike](https://github.com/collado-mike)
+* Add `Created by` view for dataset versions along with SQL syntax highlighting in web UI [#1929](https://github.com/MarquezProject/marquez/pull/1929) [@phixMe](https://github.com/phixMe)
+* Add `operationId` to openapi spec [#1978](https://github.com/MarquezProject/marquez/pull/1978) [@phixMe](https://github.com/phixMe)
 
 ### Changed
 
-* Upgrade Flyway to v7.6.0 [@dakshin-k](https://github.com/dakshin-k)
+* Upgrade Flyway to v7.6.0 [#1974](https://github.com/MarquezProject/marquez/pull/1974) [@dakshin-k](https://github.com/dakshin-k)
+
+### Fixed
+
+* Remove size limits on namespaces, dataset names, and and source connection urls [#1925](https://github.com/MarquezProject/marquez/pull/1925) [@collado-mike](https://github.com/collado-mike)
+* Update namespace names to allow `=`, `@`, and `;` [#1936](https://github.com/MarquezProject/marquez/pull/1936) [@mobuchowski](https://github.com/mobuchowski)
+* Time duration display in web UI [#1950](https://github.com/MarquezProject/marquez/pull/1950) [@phixMe](https://github.com/phixMe)
+* Enable web UI to access API via Helm Chart [@GZack2000](https://github.com/GZack2000)
 
 ## [0.21.0](https://github.com/MarquezProject/marquez/compare/0.20.0...0.21.0) - 2022-03-03
 

README.md

@@ -24,6 +24,14 @@ Marquez is an open source **metadata service** for the **collection**, **aggrega
 
 Marquez is an [LF AI & Data Foundation](https://lfaidata.foundation/projects/marquez) incubation project under active development and we'd love your help!
 
+## Adopters
+
+Want to be added? Send a pull request our way!
+
+* [Astronomer](https://astronomer.io)
+* [Datakin](https://datakin.com)
+* [Northwestern Mutual](https://www.northwesternmutual.com)
+
 ## Try it!
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/MarquezProject/marquez)
@@ -38,6 +46,8 @@ $ ./docker/up.sh
 
 > **Tip:** Use the `--build` flag to build images from source, and/or `--seed` to start Marquez with sample lineage metadata. For a more complete example using the sample metadata, please follow our [quickstart](https://marquezproject.github.io/marquez/quickstart.html) guide.
 
+> **Note:** Port 5000 is now reserved for MacOS. If running locally on MacOS, you can run `./docker/up.sh --api-port 9000` to configure the API to listen on port 9000 instead. Keep in mind that you will need to update the URLs below with the appropriate port number.
+
 **`WEB UI`**
 
 You can open [http://localhost:3000](http://localhost:3000) to begin exploring the Marquez Web UI. The UI enables you to discover dependencies between jobs and the datasets they produce and consume via the lineage graph, view run metadata of current and previous job runs, and much more!

api/build.gradle

@@ -47,7 +47,7 @@ dependencies {
     implementation "io.sentry:sentry:${sentryVersion}"
     implementation 'org.flywaydb:flyway-core:8.5.10'
     implementation "org.postgresql:postgresql:${postgresqlVersion}"
-    implementation 'com.graphql-java:graphql-java:18.0'
+    implementation 'com.graphql-java:graphql-java:18.1'
     implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
 
     testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"

api/src/main/java/marquez/cli/SeedCommand.java

@@ -53,7 +53,7 @@
  * --port}. This command is meant to be used to explore the features of Marquez. For example,
  * lineage graph, dataset schemas, job run history, etc.
  *
- * <h3>Usage</h3>
+ * <h2>Usage</h2>
  *
  * For example, to override the {@code port}:
  *

api/src/main/resources/marquez/db/migration/V43__alter_jobs_add_job_parent_uuid.sql

@@ -0,0 +1,44 @@
+ALTER TABLE jobs ADD COLUMN IF NOT EXISTS parent_job_uuid uuid CONSTRAINT jobs_parent_fk_jobs REFERENCES jobs (uuid);
+ALTER TABLE runs ADD COLUMN IF NOT EXISTS parent_run_uuid uuid CONSTRAINT runs_parent_fk_runs REFERENCES runs (uuid);
+
+DROP INDEX IF EXISTS jobs_name_index;
+ALTER TABLE jobs DROP CONSTRAINT jobs_namespace_uuid_name_key;
+DROP INDEX IF EXISTS jobs_namespace_uuid_name_key;
+
+CREATE UNIQUE INDEX IF NOT EXISTS jobs_name_parent ON jobs (name, namespace_name, parent_job_uuid);
+CREATE UNIQUE INDEX IF NOT EXISTS jobs_namespace_uuid_name_parent ON jobs (name, namespace_uuid, parent_job_uuid);
+CREATE UNIQUE INDEX IF NOT EXISTS jobs_namespace_uuid_name_null_parent ON jobs (name, namespace_uuid) WHERE parent_job_uuid IS NULL;
+ALTER TABLE jobs ADD CONSTRAINT unique_jobs_namespace_uuid_name_parent UNIQUE USING INDEX jobs_namespace_uuid_name_parent;
+
+CREATE OR REPLACE VIEW jobs_view
+AS
+    with recursive
+     job_fqn AS (
+        SELECT uuid, name, namespace_name, parent_job_uuid
+        FROM jobs
+        UNION ALL
+        SELECT j1.uuid,
+               CASE WHEN j2.name IS NOT NULL THEN j2.name || '.' || j1.name ELSE j1.name END AS name,
+               CASE WHEN j2.namespace_name IS NOT NULL THEN j2.namespace_name ELSE j1.namespace_name END AS namespace_name,
+               j2.parent_job_uuid
+        FROM jobs j1
+        INNER JOIN job_fqn j2 ON j2.uuid=j1.parent_job_uuid
+    )
+    SELECT f.uuid,
+           f.name,
+           f.namespace_name,
+           j.name AS simple_name,
+           f.parent_job_uuid,
+           j.type,
+           j.created_at,
+           j.updated_at,
+           j.namespace_uuid,
+           j.description,
+           j.current_version_uuid,
+           j.current_job_context_uuid,
+           j.current_location,
+           j.current_inputs,
+           j.symlink_target_uuid
+           FROM job_fqn f, jobs j
+           WHERE j.uuid=f.uuid
+           AND f.parent_job_uuid IS NULL

build.gradle

@@ -21,7 +21,7 @@ buildscript {
     dependencies {
         classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0'
         classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
-        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.5.1'
+        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.6.1'
     }
 }
 
@@ -54,14 +54,14 @@ subprojects {
 
     ext {
         assertjVersion = '3.22.0'
-        dropwizardVersion = '2.0.29'
+        dropwizardVersion = '2.1.0'
         jacocoVersion = '0.8.7'
         junit5Version = '5.8.2'
         lombokVersion = '1.18.24'
         mockitoVersion = '4.5.1'
         openlineageVersion = '0.8.1'
         slf4jVersion = '1.7.36'
-        postgresqlVersion = '42.3.4'
+        postgresqlVersion = '42.3.5'
         isReleaseVersion = !version.endsWith('SNAPSHOT')
     }
 

chart/Chart.yaml

@@ -29,4 +29,4 @@ name: marquez
 sources:
   - https://github.com/MarquezProject/marquez
   - https://marquezproject.github.io/marquez/
-version: 0.21.10
+version: 0.22.0

chart/values.yaml

@@ -17,7 +17,7 @@ marquez:
   image:
     registry: docker.io
     repository: marquezproject/marquez
-    tag: 0.21.0
+    tag: 0.22.0
     pullPolicy: IfNotPresent
   ## Name of the existing secret containing credentials for the Marquez installation.
   ## When this is specified, it will take precedence over the values configured in the 'db' section.
@@ -75,7 +75,7 @@ web:
   image:
     registry: docker.io
     repository: marquezproject/marquez-web
-    tag: 0.21.0
+    tag: 0.22.0
     pullPolicy: IfNotPresent
   ## Marquez website will run on this port
   ##
@@ -107,7 +107,7 @@ postgresql:
   ## @param image.tag PostgreSQL image tag (immutable tags are recommended)
   ##
   image:
-    tag: 12.1.0
+    tag: 0.22.0
   ## Authentication parameters
   ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run
   ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-on-first-run

clients/java/README.md

@@ -10,14 +10,14 @@ Maven:
 <dependency>
     <groupId>io.github.marquezproject</groupId>
     <artifactId>marquez-java</artifactId>
-    <version>0.21.0</version>
+    <version>0.22.0</version>
 </dependency>
 ```
 
 or Gradle:
 
 ```groovy
-implementation 'io.github.marquezproject:marquez-java:0.21.0
+implementation 'io.github.marquezproject:marquez-java:0.22.0
 ```
 
 ## Usage