Skip to content

Commit 7792fd5

Browse files
author
Peter Hicks
committed
Merge branch 'feature/doc-operations' of https://github.com/MarquezProject/marquez into feature/doc-operations
2 parents 6c58862 + 2618862 commit 7792fd5

43 files changed

Lines changed: 912 additions & 235 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.circleci/config.yml

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
build-api:
1919
working_directory: ~/marquez
2020
machine:
21-
image: ubuntu-2004:202010-01
21+
image: ubuntu-2004:current
2222
environment:
2323
TESTCONTAINERS_RYUK_DISABLED: true
2424
steps:
@@ -44,7 +44,8 @@ jobs:
4444

4545
build-image-api:
4646
working_directory: ~/marquez
47-
machine: true
47+
machine:
48+
image: ubuntu-2004:current
4849
steps:
4950
- checkout
5051
- run: docker build --no-cache --tag "marquezproject/marquez:${CIRCLE_SHA1}" .
@@ -54,7 +55,8 @@ jobs:
5455

5556
build-image-web:
5657
working_directory: ~/marquez/web
57-
machine: true
58+
machine:
59+
image: ubuntu-2004:current
5860
steps:
5961
- *checkout_project_root
6062
- run: docker build --no-cache --tag "marquezproject/marquez-web:${CIRCLE_SHA1}" .
@@ -85,7 +87,7 @@ jobs:
8587
build-client-java:
8688
working_directory: ~/marquez
8789
machine:
88-
image: ubuntu-2004:202010-01
90+
image: ubuntu-2004:current
8991
steps:
9092
- checkout
9193
- restore_cache:
@@ -142,7 +144,7 @@ jobs:
142144
release-java:
143145
working_directory: ~/marquez
144146
machine:
145-
image: ubuntu-2004:202010-01
147+
image: ubuntu-2004:current
146148
steps:
147149
- checkout
148150
- run: ./.circleci/get-jdk17.sh
@@ -165,7 +167,8 @@ jobs:
165167

166168
release-docker:
167169
working_directory: ~/marquez
168-
machine: true
170+
machine:
171+
image: ubuntu-2004:current
169172
steps:
170173
- checkout
171174
- run: ./docker/login.sh

.github/workflows/test-chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
fetch-depth: 0
1717

1818
- name: Setup Helm
19-
uses: azure/setup-helm@v2.0
19+
uses: azure/setup-helm@v2.1
2020

2121
- name: Setup Python
2222
uses: actions/setup-python@v3

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66

77
* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
88
* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [@wslulciuc](https://github.com/wslulciuc)
9+
* Add support for job renaming/redirection via symlink [@collado-mike](https://github.com/collado-mike)
10+
11+
### Changed
12+
13+
* Upgrade Flyway to v7.6.0 [@dakshin-k](https://github.com/dakshin-k)
914

1015
## [0.21.0](https://github.com/MarquezProject/marquez/compare/0.20.0...0.21.0) - 2022-03-03
1116

RELEASING.md

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,16 @@
1818
6. Before closing the project board for the _current_ release, move any open issues to the project board created in **step 5**
1919
7. Draft a [new release](https://github.com/MarquezProject/marquez/releases/new) using the release notes for `X.Y.Z` in **step 1** as the release description:
2020

21-
![](./docs/assets/images/new-release.png)
21+
![](./docs/assets/images/new-release.png)
22+
23+
# Voting on Releases
24+
25+
Anyone may request a new release of the project in the #general Slack channel.
26+
27+
After one is proposed, committers have 48 hours to give a +1 or -1.
28+
29+
A total of three +1s, taking into account -1s and excluding votes by the proposer, authorize the release.
30+
31+
Alternatively, if after 2 days the release has received at least one +1 and no -1s, the release is also authorized.
32+
33+
If the proposed release receives no +1s in two days, it is not authorized and the proposer must make a new request to reset the clock.

api/build.gradle

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ plugins {
2323
ext {
2424
jdbi3Version = '3.28.0'
2525
prometheusVersion = '0.15.0'
26-
testcontainersVersion = '1.16.3'
27-
sentryVersion = '5.7.2'
26+
testcontainersVersion = '1.17.1'
27+
sentryVersion = '5.7.3'
2828
}
2929

3030
dependencies {
@@ -43,9 +43,9 @@ dependencies {
4343
implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
4444
implementation "org.jdbi:jdbi3-sqlobject:${jdbi3Version}"
4545
implementation 'com.google.guava:guava:31.1-jre'
46-
implementation 'org.dhatim:dropwizard-sentry:2.0.28-10'
46+
implementation 'org.dhatim:dropwizard-sentry:2.0.29'
4747
implementation "io.sentry:sentry:${sentryVersion}"
48-
implementation 'org.flywaydb:flyway-core:6.5.7'
48+
implementation 'org.flywaydb:flyway-core:8.5.10'
4949
implementation "org.postgresql:postgresql:${postgresqlVersion}"
5050
implementation 'com.graphql-java:graphql-java:18.0'
5151
implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'

api/src/main/java/marquez/MarquezApp.java

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import javax.sql.DataSource;
2424
import lombok.NonNull;
2525
import lombok.extern.slf4j.Slf4j;
26+
import marquez.api.filter.JobRedirectFilter;
2627
import marquez.cli.SeedCommand;
2728
import marquez.common.Utils;
2829
import marquez.db.DbMigration;
@@ -115,9 +116,10 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
115116
env.jersey().register(new TracingContainerResponseFilter());
116117
}
117118

118-
registerResources(config, env, source);
119+
MarquezContext marquezContext = buildMarquezContext(config, env, (ManagedDataSource) source);
120+
registerResources(config, env, marquezContext);
119121
registerServlets(env);
120-
registerFilters(env);
122+
registerFilters(env, marquezContext);
121123
}
122124

123125
private boolean isSentryEnabled(MarquezConfig config) {
@@ -126,11 +128,26 @@ private boolean isSentryEnabled(MarquezConfig config) {
126128
}
127129

128130
public void registerResources(
129-
@NonNull MarquezConfig config, @NonNull Environment env, @NonNull DataSource source) {
131+
@NonNull MarquezConfig config, @NonNull Environment env, MarquezContext context) {
132+
133+
if (config.getGraphql().isEnabled()) {
134+
env.servlets()
135+
.addServlet("api/v1-beta/graphql", context.getGraphqlServlet())
136+
.addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
137+
}
138+
139+
log.debug("Registering resources...");
140+
for (final Object resource : context.getResources()) {
141+
env.jersey().register(resource);
142+
}
143+
}
144+
145+
private MarquezContext buildMarquezContext(
146+
MarquezConfig config, Environment env, ManagedDataSource source) {
130147
final JdbiFactory factory = new JdbiFactory();
131148
final Jdbi jdbi =
132149
factory
133-
.build(env, config.getDataSourceFactory(), (ManagedDataSource) source, DB_POSTGRES)
150+
.build(env, config.getDataSourceFactory(), source, DB_POSTGRES)
134151
.installPlugin(new SqlObjectPlugin())
135152
.installPlugin(new PostgresPlugin());
136153
SqlLogger sqlLogger = new InstrumentedSqlLogger(env.metrics());
@@ -141,17 +158,7 @@ public void registerResources(
141158

142159
final MarquezContext context =
143160
MarquezContext.builder().jdbi(jdbi).tags(config.getTags()).build();
144-
145-
if (config.getGraphql().isEnabled()) {
146-
env.servlets()
147-
.addServlet("api/v1-beta/graphql", context.getGraphqlServlet())
148-
.addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
149-
}
150-
151-
log.debug("Registering resources...");
152-
for (final Object resource : context.getResources()) {
153-
env.jersey().register(resource);
154-
}
161+
return context;
155162
}
156163

157164
private void registerServlets(@NonNull Environment env) {
@@ -161,7 +168,10 @@ private void registerServlets(@NonNull Environment env) {
161168
env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
162169
}
163170

164-
private void registerFilters(@NonNull Environment env) {
171+
private void registerFilters(@NonNull Environment env, MarquezContext marquezContext) {
165172
env.jersey().getResourceConfig().register(new LoggingMdcFilter());
173+
env.jersey()
174+
.getResourceConfig()
175+
.register(new JobRedirectFilter(marquezContext.getJobService()));
166176
}
167177
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package marquez.api.filter;
2+
3+
import java.io.IOException;
4+
import java.net.URI;
5+
import java.util.HashMap;
6+
import java.util.List;
7+
import java.util.Map;
8+
import java.util.Optional;
9+
import javax.ws.rs.container.ContainerRequestContext;
10+
import javax.ws.rs.container.ContainerRequestFilter;
11+
import javax.ws.rs.core.MultivaluedHashMap;
12+
import javax.ws.rs.core.MultivaluedMap;
13+
import javax.ws.rs.core.Response;
14+
import javax.ws.rs.core.UriBuilder;
15+
import lombok.extern.slf4j.Slf4j;
16+
import marquez.common.models.JobId;
17+
import marquez.db.models.JobRow;
18+
import marquez.service.JobService;
19+
import marquez.service.models.Job;
20+
import org.glassfish.jersey.server.ExtendedUriInfo;
21+
import org.glassfish.jersey.uri.UriComponent;
22+
import org.glassfish.jersey.uri.UriComponent.Type;
23+
import org.glassfish.jersey.uri.UriTemplate;
24+
25+
/**
26+
* Filters requests that reference a job that has been symlinked to another job. This filter
27+
* redirects such requests to the URL with the symlink target's name using a 301 status code.
28+
*/
29+
@Slf4j
30+
public class JobRedirectFilter implements ContainerRequestFilter {
31+
32+
public static final String JOB_PATH_PARAM = "job";
33+
public static final String NAMESPACE_PATH_PARAM = "namespace";
34+
private final JobService jobService;
35+
36+
public JobRedirectFilter(JobService jobService) {
37+
this.jobService = jobService;
38+
}
39+
40+
@Override
41+
public void filter(ContainerRequestContext requestContext) throws IOException {
42+
MultivaluedMap<String, String> pathParams = requestContext.getUriInfo().getPathParameters();
43+
if (!pathParams.containsKey(NAMESPACE_PATH_PARAM) || !pathParams.containsKey(JOB_PATH_PARAM)) {
44+
return;
45+
}
46+
List<String> namespaceParams = pathParams.get(NAMESPACE_PATH_PARAM);
47+
List<String> jobParams = pathParams.get(JOB_PATH_PARAM);
48+
if (namespaceParams.isEmpty() || jobParams.isEmpty()) {
49+
return;
50+
}
51+
Optional<Job> job = jobService.findJobByName(namespaceParams.get(0), jobParams.get(0));
52+
job.ifPresent(
53+
j -> {
54+
if (!j.getName().getValue().equals(jobParams.get(0))) {
55+
log.info(
56+
"Job {}.{} has been redirected to {}.{}",
57+
namespaceParams.get(0),
58+
jobParams.get(0),
59+
j.getNamespace().getValue(),
60+
j.getName().getValue());
61+
URI location = buildLocationFor(requestContext, j.getId());
62+
log.debug("Redirecting to url {}", location);
63+
requestContext.abortWith(Response.status(301).location(location).build());
64+
}
65+
});
66+
}
67+
68+
/**
69+
* Construct a URI from a Request's matched resource, replacing the {@value #JOB_PATH_PARAM} and
70+
* {@value #NAMESPACE_PATH_PARAM} parameters with the fully-qualified values from the provided
71+
* {@link JobRow}.
72+
*
73+
* @param ctx
74+
* @param jobId
75+
* @return
76+
*/
77+
private URI buildLocationFor(ContainerRequestContext ctx, JobId jobId) {
78+
Object resource = ctx.getUriInfo().getMatchedResources().get(0);
79+
MultivaluedMap<String, String> pathParameters = ctx.getUriInfo().getPathParameters();
80+
MultivaluedHashMap<String, String> copy = new MultivaluedHashMap<>(pathParameters);
81+
copy.putSingle(
82+
JOB_PATH_PARAM, UriComponent.encode(jobId.getName().getValue(), Type.PATH_SEGMENT));
83+
copy.putSingle(
84+
NAMESPACE_PATH_PARAM,
85+
UriComponent.encode(jobId.getNamespace().getValue(), Type.PATH_SEGMENT));
86+
Map<String, String> singletonMap = new HashMap<>();
87+
copy.forEach((k, v) -> singletonMap.put(k, v.get(0)));
88+
UriTemplate pathTemplate = ((ExtendedUriInfo) ctx.getUriInfo()).getMatchedTemplates().get(0);
89+
String newPath = pathTemplate.createURI(singletonMap);
90+
return UriBuilder.fromResource(resource.getClass()).path(newPath).buildFromEncodedMap(copy);
91+
}
92+
}

api/src/main/java/marquez/common/models/NamespaceName.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,15 @@ public final class NamespaceName {
2121
private static final int MIN_SIZE = 1;
2222
private static final int MAX_SIZE = 1024;
2323
private static final Pattern PATTERN =
24-
Pattern.compile(String.format("^[a-zA-Z:;=/0-9_\\-\\.]{%d,%d}$", MIN_SIZE, MAX_SIZE));
24+
Pattern.compile(String.format("^[a-zA-Z:;=/0-9_\\-\\.@]{%d,%d}$", MIN_SIZE, MAX_SIZE));
2525

2626
@Getter private final String value;
2727

2828
public NamespaceName(@NonNull final String value) {
2929
checkArgument(
3030
PATTERN.matcher(value).matches(),
3131
"namespace '%s' must contain only letters (a-z, A-Z), numbers (0-9), "
32-
+ "underscores (_), dashes (-), colons (:), equals (=), semicolons (;), slashes (/) "
32+
+ "underscores (_), at (@), dashes (-), colons (:), equals (=), semicolons (;), slashes (/) "
3333
+ "or dots (.) with a maximum length of %s characters.",
3434
value,
3535
MAX_SIZE);

api/src/main/java/marquez/db/Columns.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
@Slf4j
2727
public final class Columns {
28+
2829
private Columns() {}
2930

3031
private static final ObjectMapper MAPPER = Utils.getMapper();
@@ -78,6 +79,9 @@ private Columns() {}
7879
/* STREAM VERSION ROW COLUMNS */
7980
public static final String SCHEMA_LOCATION = "schema_location";
8081

82+
/* JOB ROW COLUMNS */
83+
public static final String SYMLINK_TARGET_UUID = "symlink_target_uuid";
84+
8185
/* JOB VERSION I/O ROW COLUMNS */
8286
public static final String INPUT_UUIDS = "input_uuids";
8387
public static final String OUTPUT_UUIDS = "output_uuids";

api/src/main/java/marquez/db/DbMigration.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import lombok.extern.slf4j.Slf4j;
88
import org.flywaydb.core.Flyway;
99
import org.flywaydb.core.api.FlywayException;
10+
import org.flywaydb.core.api.output.MigrateResult;
1011

1112
@Slf4j
1213
public final class DbMigration {
@@ -31,8 +32,9 @@ public static void migrateDbOrError(
3132
// issues before app termination.
3233
try {
3334
log.info("Migrating database...");
34-
final int migrations = flyway.migrate();
35-
log.info("Successfully applied '{}' migrations to database.", migrations);
35+
final MigrateResult migrateResult = flyway.migrate();
36+
log.info(
37+
"Successfully applied '{}' migrations to database.", migrateResult.migrationsExecuted);
3638
} catch (FlywayException errorOnDbMigrate) {
3739
log.error("Failed to apply migration to database.", errorOnDbMigrate);
3840
try {

0 commit comments

Comments
 (0)