Merge branch 'main' into column-lineage-proposal

Author: pawel-big-lebowski

.github/boring-cyborg.yml

@@ -0,0 +1,58 @@
+##### Labeler ##########################################################################################################
+# Enable "labeler" for your PR that would add labels to PRs based on the paths that are modified in the PR.
+labelPRBasedOnFilePath:
+  # Add 'api' to any changes within 'api' folder or any subfolders
+    api:
+      - api/**/*
+
+  # Add 'chart' to any changes within 'chart' folder or any subfolders
+    chart:
+      - chart/**/*
+
+  # Add 'docs' to any changes to `.md` files in root or within 'docs' folder, subfolders
+    docs:
+      - docs/**/*
+      - "*.md"
+
+  # Add 'client/java' to any changes within 'clients/java' folder or any subfolders
+    client/java:
+      - clients/java/**/*
+
+  # Add 'client/python' to any changes within 'clients/python' folder or any subfolders
+    client/python:
+      - clients/python/**/*
+
+  # Add 'docker' to any changes within 'docker' folder
+    docker:
+      - docker/*
+
+  # Add 'example' to any changes within 'examples' folder or any subfolders
+    example:
+      - examples/**/*
+
+  # Add 'proposal' to any changes within 'proposals' folder or any subfolders
+    proposal:
+      - proposals/**/*
+
+  # Add 'spec' to any changes within 'spec' folder
+    spec:
+      - spec/*
+
+  # Add 'web' to any changes within 'web' folder or any subfolders
+    web:
+      - web/**/*
+
+
+
+##### Greetings ########################################################################################################
+# Comment to be posted to welcome users when they open their first PR
+firstPRWelcomeComment: >
+  Thanks for opening your first pull request in the Marquez project! Please check out our contributing guidelines (https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md).
+
+# Comment to be posted to congratulate user on their first merged PR
+firstPRMergeComment: >
+  Great job! Congrats on your first merged pull request in the Marquez project!
+
+# Comment to be posted to on first time issues
+firstIssueWelcomeComment: >
+  Thanks for opening your first issue in the Marquez project! Please be sure to follow the issue template!

.gitignore

@@ -32,6 +32,9 @@ venv
 # Marquez configuration
 marquez.yml
 
+# Metadata
+metadata.json
+
 # jenv
 .java-version
 

CHANGELOG.md

@@ -6,65 +6,65 @@
 
 ### Fixed
 
-* Fix py module release [#2057](https://github.com/MarquezProject/marquez/pull/2057) [@wslulciuc](https://github.com/wslulciuc)
-* Use /bin/sh in web/docker/entrypoint.sh [#2059](https://github.com/MarquezProject/marquez/pull/2059) [@wslulciuc](https://github.com/wslulciuc)
+* Fix py module release [`#2057`](https://github.com/MarquezProject/marquez/pull/2057) [@wslulciuc](https://github.com/wslulciuc)
+* Use `/bin/sh` in `web/docker/entrypoint.sh` [`#2059`](https://github.com/MarquezProject/marquez/pull/2059) [@wslulciuc](https://github.com/wslulciuc)
 
 ## [0.24.0](https://github.com/MarquezProject/marquez/compare/0.23.0...0.24.0) - 2022-08-02
 
 ### Added
 
-* Add copyright lines to all source files [#1996](https://github.com/MarquezProject/marquez/pull/1996) [@merobi-hub](https://github.com/MarquezProject/marquez/commits?author=merobi-hub)
-* Add copyright and license guidelines in CONTRIBUTING.md [@wslulciuc](https://github.com/wslulciuc)
-* Add @FlywayTarget annotation to migration tests to control flyway upgrades [#2035](https://github.com/MarquezProject/marquez/pull/2035) [@collado-mike](https://github.com/collado-mike) 
+* Add copyright lines to all source files [`#1996`](https://github.com/MarquezProject/marquez/pull/1996) [@merobi-hub](https://github.com/MarquezProject/marquez/commits?author=merobi-hub)
+* Add copyright and license guidelines in `CONTRIBUTING.md` [@wslulciuc](https://github.com/wslulciuc)
+* Add `@FlywayTarget` annotation to migration tests to control flyway upgrades [`#2035`](https://github.com/MarquezProject/marquez/pull/2035) [@collado-mike](https://github.com/collado-mike)
 
 ### Changed
 
-* Updated `jobs_view` to stop computing FQN on reads and to compute on _writes_ instead [#2036](https://github.com/MarquezProject/marquez/pull/2036) [@collado-mike](https://github.com/collado-mike)
-* Runs row reduction [#2041](https://github.com/MarquezProject/marquez/pull/2041) [@collado-mike](https://github.com/collado-mike)
+* Updated `jobs_view` to stop computing FQN on reads and to compute on _writes_ instead [`#2036`](https://github.com/MarquezProject/marquez/pull/2036) [@collado-mike](https://github.com/collado-mike)
+* Runs row reduction [`#2041`](https://github.com/MarquezProject/marquez/pull/2041) [@collado-mike](https://github.com/collado-mike)
 
 ### Fixed
 
-* Update `Run` in the openapi spec to include a `context` field [#2020](https://github.com/MarquezProject/marquez/pull/2020) [@esaych](https://github.com/Esaych)
-* Fix dataset openapi model [#2038](https://github.com/MarquezProject/marquez/pull/2038) [@esaych](https://github.com/Esaych) 
-* Fix casing on lastLifecycleState [#2039](https://github.com/MarquezProject/marquez/pull/2039) [@esaych](https://github.com/Esaych)
-* Fix V45 migration to include initial population of jobs_fqn table [#2051](https://github.com/MarquezProject/marquez/pull/2051) [@collado-mike](https://github.com/collado-mike)
-* Fix symlinked jobs in queries [#2053](https://github.com/MarquezProject/marquez/pull/2053) [@collado-mike](https://github.com/collado-mike)
+* Update `Run` in the openapi spec to include a `context` field [`#2020`](https://github.com/MarquezProject/marquez/pull/2020) [@esaych](https://github.com/Esaych)
+* Fix dataset openapi model [`#2038`](https://github.com/MarquezProject/marquez/pull/2038) [@esaych](https://github.com/Esaych)
+* Fix casing on `lastLifecycleState` [`#2039`](https://github.com/MarquezProject/marquez/pull/2039) [@esaych](https://github.com/Esaych)
+* Fix V45 migration to include initial population of jobs_fqn table [`#2051`](https://github.com/MarquezProject/marquez/pull/2051) [@collado-mike](https://github.com/collado-mike)
+* Fix symlinked jobs in queries [`#2053`](https://github.com/MarquezProject/marquez/pull/2053) [@collado-mike](https://github.com/collado-mike)
 
 ## [0.23.0](https://github.com/MarquezProject/marquez/compare/0.22.0...0.23.0) - 2022-06-16
 
 ### Added
 
-* Update docker-compose.yml: Randomly map postgres db port [#2000](https://github.com/MarquezProject/marquez/pull/2000) [@RNHTTR](https://github.com/RNHTTR) 
-* Job parent hierarchy [#1935](https://github.com/MarquezProject/marquez/pull/1935) [#1980](https://github.com/MarquezProject/marquez/pull/1980) [#1992](https://github.com/MarquezProject/marquez/pull/1992) [@collado-mike](https://github.com/collado-mike)
+* Update docker-compose.yml: Randomly map postgres db port [`#2000`](https://github.com/MarquezProject/marquez/pull/2000) [@RNHTTR](https://github.com/RNHTTR)
+* Job parent hierarchy [`#1935`](https://github.com/MarquezProject/marquez/pull/1935) [`#1980`](https://github.com/MarquezProject/marquez/pull/1980) [`#1992`](https://github.com/MarquezProject/marquez/pull/1992) [@collado-mike](https://github.com/collado-mike)
 
 ### Changed
 
-* Set default limit for listing datasets and jobs in UI from `2000` to `25` [#2018](https://github.com/MarquezProject/marquez/pull/2018) [@wslulciuc](https://github.com/wslulciuc)
+* Set default limit for listing datasets and jobs in UI from `2000` to `25` [`#2018`](https://github.com/MarquezProject/marquez/pull/2018) [@wslulciuc](https://github.com/wslulciuc)
 * Update OpenLineage write API to be non-transactional and avoid unnecessary locks on records under heavy contention [@collado-mike](https://github.com/collado-mike)
 
 ### Fixed
 
-* Return the tag for postgresql to 12.1.0 [#2015](https://github.com/MarquezProject/marquez/pull/2015) [@rossturk](https://github.com/rossturk)
+* Return the tag for postgresql to 12.1.0 [`#2015`](https://github.com/MarquezProject/marquez/pull/2015) [@rossturk](https://github.com/rossturk)
 
 ## [0.22.0](https://github.com/MarquezProject/marquez/compare/0.21.0...0.22.0) - 2022-05-16
 
 ### Added
 
-* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [#1847](https://github.com/MarquezProject/marquez/pull/1847)[@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
-* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [#1945](https://github.com/MarquezProject/marquez/pull/1945) [@wslulciuc](https://github.com/wslulciuc)
-* Add support for job renaming/redirection via symlink [#1947](https://github.com/MarquezProject/marquez/pull/1947) [@collado-mike](https://github.com/collado-mike)
-* Add `Created by` view for dataset versions along with SQL syntax highlighting in web UI [#1929](https://github.com/MarquezProject/marquez/pull/1929) [@phixMe](https://github.com/phixMe)
-* Add `operationId` to openapi spec [#1978](https://github.com/MarquezProject/marquez/pull/1978) [@phixMe](https://github.com/phixMe)
+* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [`#1847`](https://github.com/MarquezProject/marquez/pull/1847)[@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
+* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [`#1945`](https://github.com/MarquezProject/marquez/pull/1945) [@wslulciuc](https://github.com/wslulciuc)
+* Add support for job renaming/redirection via symlink [`#1947`](https://github.com/MarquezProject/marquez/pull/1947) [@collado-mike](https://github.com/collado-mike)
+* Add `Created by` view for dataset versions along with SQL syntax highlighting in web UI [`#1929`](https://github.com/MarquezProject/marquez/pull/1929) [@phixMe](https://github.com/phixMe)
+* Add `operationId` to openapi spec [`#1978`](https://github.com/MarquezProject/marquez/pull/1978) [@phixMe](https://github.com/phixMe)
 
 ### Changed
 
-* Upgrade Flyway to v7.6.0 [#1974](https://github.com/MarquezProject/marquez/pull/1974) [@dakshin-k](https://github.com/dakshin-k)
+* Upgrade Flyway to v7.6.0 [`#1974`](https://github.com/MarquezProject/marquez/pull/1974) [@dakshin-k](https://github.com/dakshin-k)
 
 ### Fixed
 
-* Remove size limits on namespaces, dataset names, and and source connection urls [#1925](https://github.com/MarquezProject/marquez/pull/1925) [@collado-mike](https://github.com/collado-mike)
-* Update namespace names to allow `=`, `@`, and `;` [#1936](https://github.com/MarquezProject/marquez/pull/1936) [@mobuchowski](https://github.com/mobuchowski)
-* Time duration display in web UI [#1950](https://github.com/MarquezProject/marquez/pull/1950) [@phixMe](https://github.com/phixMe)
+* Remove size limits on namespaces, dataset names, and and source connection urls [`#1925`](https://github.com/MarquezProject/marquez/pull/1925) [@collado-mike](https://github.com/collado-mike)
+* Update namespace names to allow `=`, `@`, and `;` [`#1936`](https://github.com/MarquezProject/marquez/pull/1936) [@mobuchowski](https://github.com/mobuchowski)
+* Time duration display in web UI [`#1950`](https://github.com/MarquezProject/marquez/pull/1950) [@phixMe](https://github.com/phixMe)
 * Enable web UI to access API via Helm Chart [@GZack2000](https://github.com/GZack2000)
 
 ## [0.21.0](https://github.com/MarquezProject/marquez/compare/0.20.0...0.21.0) - 2022-03-03

RELEASING.md

@@ -15,6 +15,10 @@
 
    ![](./docs/assets/images/new-release.png)
 
+# Release Cadence
+
+A release will be initiated on the 15th day of each month unless this falls on a Friday, Saturday, Sunday or US holiday, in which case the release will commence on the following business day.
+
 # Voting on Releases
 
 Anyone may request a new release of the project in the #general Slack channel.

api/build.gradle

@@ -21,10 +21,10 @@ plugins {
 }
 
 ext {
-    jdbi3Version = '3.28.0'
+    jdbi3Version = '3.32.0'
     prometheusVersion = '0.15.0'
-    testcontainersVersion = '1.17.1'
-    sentryVersion = '5.7.3'
+    testcontainersVersion = '1.17.3'
+    sentryVersion = '5.7.4'
 }
 
 dependencies {
@@ -43,9 +43,9 @@ dependencies {
     implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
     implementation "org.jdbi:jdbi3-sqlobject:${jdbi3Version}"
     implementation 'com.google.guava:guava:31.1-jre'
-    implementation 'org.dhatim:dropwizard-sentry:2.0.29'
+    implementation 'org.dhatim:dropwizard-sentry:2.1.1-5'
     implementation "io.sentry:sentry:${sentryVersion}"
-    implementation 'org.flywaydb:flyway-core:8.5.10'
+    implementation 'org.flywaydb:flyway-core:8.5.13'
     implementation "org.postgresql:postgresql:${postgresqlVersion}"
     implementation 'com.graphql-java:graphql-java:18.1'
     implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'

api/src/main/java/marquez/MarquezApp.java

@@ -27,6 +27,7 @@
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import marquez.api.filter.JobRedirectFilter;
+import marquez.cli.MetadataCommand;
 import marquez.cli.SeedCommand;
 import marquez.common.Utils;
 import marquez.db.DbMigration;
@@ -75,6 +76,7 @@ public void initialize(@NonNull Bootstrap<MarquezConfig> bootstrap) {
             new EnvironmentVariableSubstitutor(ERROR_ON_UNDEFINED)));
 
     // Add CLI commands
+    bootstrap.addCommand(new MetadataCommand());
     bootstrap.addCommand(new SeedCommand());
 
     bootstrap.getObjectMapper().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);

api/src/main/java/marquez/api/DatasetResource.java

@@ -17,6 +17,7 @@
 import javax.validation.Valid;
 import javax.validation.constraints.Min;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
@@ -150,6 +151,28 @@ public Response list(
     return Response.ok(new ResultsPage<>("datasets", datasets, totalCount)).build();
   }
 
+  @Timed
+  @ResponseMetered
+  @ExceptionMetered
+  @DELETE
+  @Path("{dataset}")
+  @Produces(APPLICATION_JSON)
+  public Response delete(
+      @PathParam("namespace") NamespaceName namespaceName,
+      @PathParam("dataset") DatasetName datasetName) {
+    throwIfNotExists(namespaceName);
+
+    Dataset dataset =
+        datasetService
+            .findDatasetByName(namespaceName.getValue(), datasetName.getValue())
+            .orElseThrow(() -> new DatasetNotFoundException(datasetName));
+
+    datasetService
+        .delete(namespaceName.getValue(), datasetName.getValue())
+        .orElseThrow(() -> new DatasetNotFoundException(datasetName));
+    return Response.ok(dataset).build();
+  }
+
   @Timed
   @ResponseMetered
   @ExceptionMetered

api/src/main/java/marquez/api/JobResource.java

@@ -16,6 +16,7 @@
 import javax.validation.Valid;
 import javax.validation.constraints.Min;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
@@ -160,6 +161,25 @@ public Response list(
     return Response.ok(new ResultsPage<>("jobs", jobs, totalCount)).build();
   }
 
+  @Timed
+  @ResponseMetered
+  @ExceptionMetered
+  @DELETE
+  @Path("/namespaces/{namespace}/jobs/{job}")
+  @Produces(APPLICATION_JSON)
+  public Response delete(
+      @PathParam("namespace") NamespaceName namespaceName, @PathParam("job") JobName jobName) {
+    throwIfNotExists(namespaceName);
+
+    Job job =
+        jobService
+            .findJobByName(namespaceName.getValue(), jobName.getValue())
+            .orElseThrow(() -> new JobNotFoundException(jobName));
+
+    jobService.delete(namespaceName.getValue(), jobName.getValue());
+    return Response.ok(job).build();
+  }
+
   @Timed
   @ResponseMetered
   @ExceptionMetered