Skip to content

Commit d9daa4e

Browse files
JDarDagranJDarDagran
committed
Remove job context.
Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Add call to API in frontend. Add tests. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Update javadocs. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Apply spotless fixes. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com>
1 parent 3492df2 commit d9daa4e

76 files changed

Lines changed: 499 additions & 481 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/java/marquez/MarquezContext.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import lombok.NonNull;
1616
import marquez.api.ColumnLineageResource;
1717
import marquez.api.DatasetResource;
18+
import marquez.api.FacetResource;
1819
import marquez.api.JobResource;
1920
import marquez.api.NamespaceResource;
2021
import marquez.api.OpenLineageResource;
@@ -28,7 +29,6 @@
2829
import marquez.db.DatasetDao;
2930
import marquez.db.DatasetFieldDao;
3031
import marquez.db.DatasetVersionDao;
31-
import marquez.db.JobContextDao;
3232
import marquez.db.JobDao;
3333
import marquez.db.JobVersionDao;
3434
import marquez.db.LineageDao;
@@ -67,7 +67,6 @@ public final class MarquezContext {
6767
@Getter private final DatasetVersionDao datasetVersionDao;
6868
@Getter private final JobDao jobDao;
6969
@Getter private final JobVersionDao jobVersionDao;
70-
@Getter private final JobContextDao jobContextDao;
7170
@Getter private final RunDao runDao;
7271
@Getter private final RunArgsDao runArgsDao;
7372
@Getter private final RunStateDao runStateDao;
@@ -91,6 +90,7 @@ public final class MarquezContext {
9190
@Getter private final SourceResource sourceResource;
9291
@Getter private final DatasetResource datasetResource;
9392
@Getter private final ColumnLineageResource columnLineageResource;
93+
@Getter private final FacetResource facetResource;
9494
@Getter private final JobResource jobResource;
9595
@Getter private final TagResource tagResource;
9696
@Getter private final OpenLineageResource openLineageResource;
@@ -116,7 +116,6 @@ private MarquezContext(
116116
this.datasetVersionDao = jdbi.onDemand(DatasetVersionDao.class);
117117
this.jobDao = jdbi.onDemand(JobDao.class);
118118
this.jobVersionDao = jdbi.onDemand(JobVersionDao.class);
119-
this.jobContextDao = jdbi.onDemand(JobContextDao.class);
120119
this.runDao = jdbi.onDemand(RunDao.class);
121120
this.runArgsDao = jdbi.onDemand(RunArgsDao.class);
122121
this.runStateDao = jdbi.onDemand(RunStateDao.class);
@@ -158,6 +157,7 @@ private MarquezContext(
158157
this.sourceResource = new SourceResource(serviceFactory);
159158
this.datasetResource = new DatasetResource(serviceFactory);
160159
this.columnLineageResource = new ColumnLineageResource(serviceFactory);
160+
this.facetResource = new FacetResource(serviceFactory, openLineageDao);
161161
this.jobResource = new JobResource(serviceFactory, jobVersionDao);
162162
this.tagResource = new TagResource(serviceFactory);
163163
this.openLineageResource = new OpenLineageResource(serviceFactory, openLineageDao);
@@ -168,6 +168,7 @@ private MarquezContext(
168168
namespaceResource,
169169
sourceResource,
170170
datasetResource,
171+
facetResource,
171172
columnLineageResource,
172173
jobResource,
173174
tagResource,

‎api/src/main/java/marquez/api/FacetResource.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2018-2023 contributors to the Marquez project
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package marquez.api;
7+
8+
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
9+
10+
import com.codahale.metrics.annotation.ExceptionMetered;
11+
import com.codahale.metrics.annotation.ResponseMetered;
12+
import com.codahale.metrics.annotation.Timed;
13+
import javax.ws.rs.GET;
14+
import javax.ws.rs.Path;
15+
import javax.ws.rs.Produces;
16+
import javax.ws.rs.PathParam;
17+
import javax.ws.rs.core.Response;
18+
import lombok.NonNull;
19+
import lombok.extern.slf4j.Slf4j;
20+
import marquez.common.models.RunId;
21+
import marquez.db.OpenLineageDao;
22+
import marquez.service.ServiceFactory;
23+
import marquez.service.models.Facets;
24+
25+
26+
@Slf4j
27+
@Path("/api/v1/facets")
28+
public class FacetResource extends BaseResource {
29+
30+
private final OpenLineageDao openLineageDao;
31+
32+
public FacetResource(
33+
@NonNull final ServiceFactory serviceFactory, @NonNull final OpenLineageDao openLineageDao) {
34+
super(serviceFactory);
35+
this.openLineageDao = openLineageDao;
36+
}
37+
38+
@Timed
39+
@ResponseMetered
40+
@ExceptionMetered
41+
@GET
42+
@Produces(APPLICATION_JSON)
43+
@Path("/run/{id}")
44+
public Response getRunFacets(
45+
@PathParam("id") RunId runId) {
46+
throwIfNotExists(runId);
47+
Facets runFacets = openLineageDao.findRunFacetsByRunUuid(runId.getValue());
48+
return Response.ok(runFacets).build();
49+
}
50+
51+
@Timed
52+
@ResponseMetered
53+
@ExceptionMetered
54+
@GET
55+
@Produces(APPLICATION_JSON)
56+
@Path("job/run/{id}")
57+
public Response getJobFacets(
58+
@PathParam("id") RunId runId) {
59+
throwIfNotExists(runId);
60+
Facets jobFacets = openLineageDao.findJobFacetsByRunUuid(runId.getValue());
61+
return Response.ok(jobFacets).build();
62+
}
63+
}

‎api/src/main/java/marquez/api/models/JobVersion.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
package marquez.api.models;
77

8-
import com.google.common.collect.ImmutableMap;
98
import java.net.URL;
109
import java.time.Instant;
1110
import java.util.List;
@@ -35,7 +34,6 @@ public final class JobVersion {
3534
@Getter private final Version version;
3635
@Getter private final NamespaceName namespace;
3736
@Nullable private final URL location;
38-
@Getter private final ImmutableMap<String, String> context;
3937
@Getter private final List<DatasetId> inputs;
4038
@Getter private final List<DatasetId> outputs;
4139
@Getter @Nullable private final Run latestRun;
@@ -46,7 +44,6 @@ public JobVersion(
4644
@NonNull final Instant createdAt,
4745
@NonNull final Version version,
4846
@Nullable final URL location,
49-
@Nullable final ImmutableMap<String, String> context,
5047
List<DatasetId> inputs,
5148
List<DatasetId> outputs,
5249
@Nullable Run latestRun) {
@@ -56,7 +53,6 @@ public JobVersion(
5653
this.version = version;
5754
this.namespace = id.getNamespace();
5855
this.location = location;
59-
this.context = (context == null) ? ImmutableMap.of() : context;
6056
this.inputs = inputs;
6157
this.outputs = outputs;
6258
this.latestRun = latestRun;

‎api/src/main/java/marquez/common/Utils.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.fasterxml.jackson.databind.SerializationFeature;
2121
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2222
import com.google.common.base.Joiner;
23-
import com.google.common.collect.ImmutableMap;
2423
import com.google.common.collect.ImmutableSet;
2524
import com.google.common.hash.Hashing;
2625
import io.dropwizard.jackson.Jackson;
@@ -236,7 +235,6 @@ public static Instant toInstant(@Nullable final String asIso) {
236235
* @param jobName The name of the job.
237236
* @param jobInputIds The input dataset IDs for the job.
238237
* @param jobOutputIds The output dataset IDs for the job.
239-
* @param jobContext The context of the job.
240238
* @param jobLocation The source code location for the job.
241239
* @return A {@link Version} object based on the specified job meta.
242240
*/
@@ -245,7 +243,6 @@ public static Version newJobVersionFor(
245243
@NonNull final JobName jobName,
246244
@NonNull final ImmutableSet<DatasetId> jobInputIds,
247245
@NonNull final ImmutableSet<DatasetId> jobOutputIds,
248-
@NonNull final ImmutableMap<String, String> jobContext,
249246
@Nullable final String jobLocation) {
250247
final byte[] bytes =
251248
VERSION_JOINER
@@ -268,8 +265,7 @@ public static Version newJobVersionFor(
268265
jobOutputId.getNamespace().getValue(),
269266
jobOutputId.getName().getValue()))
270267
.collect(joining(VERSION_DELIM)),
271-
jobLocation,
272-
KV_JOINER.join(jobContext))
268+
jobLocation)
273269
.getBytes(UTF_8);
274270
return Version.of(UUID.nameUUIDFromBytes(bytes));
275271
}

‎api/src/main/java/marquez/common/models/DatasetId.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import com.fasterxml.jackson.annotation.JsonProperty;
99
import com.google.common.collect.ComparisonChain;
10-
import com.google.common.collect.ImmutableMap;
1110
import com.google.common.collect.ImmutableSet;
1211
import lombok.EqualsAndHashCode;
1312
import lombok.NonNull;
@@ -16,8 +15,8 @@
1615

1716
/**
1817
* ID for {@code Dataset}. The class implements {@link Comparable} to ensure job versions generated
19-
* with {@link Utils#newJobVersionFor(NamespaceName, JobName, ImmutableSet, ImmutableSet,
20-
* ImmutableMap, String)} are consistent as jobs may contain inputs and outputs out of order.
18+
* with {@link Utils#newJobVersionFor(NamespaceName, JobName, ImmutableSet, ImmutableSet, String)}
19+
* are consistent as jobs may contain inputs and outputs out of order.
2120
*/
2221
@EqualsAndHashCode
2322
@ToString

‎api/src/main/java/marquez/db/BaseDao.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ public interface BaseDao extends SqlObject {
1818
@CreateSqlObject
1919
DatasetVersionDao createDatasetVersionDao();
2020

21-
@CreateSqlObject
22-
JobContextDao createJobContextDao();
23-
2421
@CreateSqlObject
2522
JobDao createJobDao();
2623

‎api/src/main/java/marquez/db/Columns.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,9 @@ private Columns() {}
103103

104104
/* JOB VERSION ROW COLUMNS */
105105
public static final String JOB_UUID = "job_uuid";
106-
public static final String JOB_CONTEXT_UUID = "job_context_uuid";
107106
public static final String LOCATION = "location";
108107
public static final String LATEST_RUN_UUID = "latest_run_uuid";
109108

110-
/* JOB CONTEXT ROW COLUMNS */
111-
public static final String CONTEXT = "context";
112-
113109
/* RUN ROW COLUMNS */
114110
public static final String EXTERNAL_ID = "external_id";
115111
public static final String RUN_ARGS_UUID = "run_args_uuid";

‎api/src/main/java/marquez/db/JobContextDao.java‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎api/src/main/java/marquez/db/JobDao.java‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,13 @@
1515
import java.util.Set;
1616
import java.util.UUID;
1717
import java.util.stream.Collectors;
18-
import marquez.common.Utils;
1918
import marquez.common.models.DatasetId;
2019
import marquez.common.models.DatasetName;
2120
import marquez.common.models.JobName;
2221
import marquez.common.models.JobType;
2322
import marquez.common.models.NamespaceName;
2423
import marquez.db.mappers.JobMapper;
2524
import marquez.db.mappers.JobRowMapper;
26-
import marquez.db.models.JobContextRow;
2725
import marquez.db.models.JobRow;
2826
import marquez.db.models.NamespaceRow;
2927
import marquez.service.models.Job;
@@ -58,10 +56,9 @@ SELECT EXISTS (
5856

5957
@SqlQuery(
6058
"""
61-
SELECT j.*, jc.context, f.facets
59+
SELECT j.*, f.facets
6260
FROM jobs_view j
6361
LEFT OUTER JOIN job_versions AS jv ON jv.uuid = j.current_version_uuid
64-
LEFT OUTER JOIN job_contexts jc ON jc.uuid = j.current_job_context_uuid
6562
LEFT OUTER JOIN (
6663
SELECT run_uuid, JSON_AGG(e.facets) AS facets
6764
FROM (
@@ -130,10 +127,9 @@ default Optional<Job> findWithRun(String namespaceName, String jobName) {
130127

131128
@SqlQuery(
132129
"""
133-
SELECT j.*, jc.context, f.facets
130+
SELECT j.*, f.facets
134131
FROM jobs_view AS j
135132
LEFT OUTER JOIN job_versions AS jv ON jv.uuid = j.current_version_uuid
136-
LEFT OUTER JOIN job_contexts jc ON jc.uuid = j.current_job_context_uuid
137133
LEFT OUTER JOIN (
138134
SELECT run_uuid, JSON_AGG(e.facets) AS facets
139135
FROM (
@@ -208,13 +204,6 @@ default JobRow upsertJobMeta(
208204
createNamespaceDao()
209205
.upsertNamespaceRow(
210206
UUID.randomUUID(), createdAt, namespaceName.getValue(), DEFAULT_NAMESPACE_OWNER);
211-
JobContextRow contextRow =
212-
createJobContextDao()
213-
.upsert(
214-
UUID.randomUUID(),
215-
createdAt,
216-
Utils.toJson(jobMeta.getContext()),
217-
Utils.checksumFor(jobMeta.getContext()));
218207
return upsertJob(
219208
UUID.randomUUID(),
220209
jobMeta.getType(),
@@ -223,7 +212,6 @@ default JobRow upsertJobMeta(
223212
namespace.getName(),
224213
jobName.getValue(),
225214
jobMeta.getDescription().orElse(null),
226-
contextRow.getUuid(),
227215
toUrlString(jobMeta.getLocation().orElse(null)),
228216
symlinkTargetUuid,
229217
toJson(jobMeta.getInputs(), mapper));
@@ -276,7 +264,7 @@ INSERT INTO jobs_view AS j (
276264
:namespaceName,
277265
:name,
278266
:description,
279-
:jobContextUuid,
267+
null,
280268
:location,
281269
:inputs,
282270
:symlinkTargetId,
@@ -291,7 +279,6 @@ JobRow upsertJob(
291279
String namespaceName,
292280
String name,
293281
String description,
294-
UUID jobContextUuid,
295282
String location,
296283
UUID symlinkTargetId,
297284
PGobject inputs);
@@ -328,7 +315,7 @@ INSERT INTO jobs_view AS j (
328315
:namespaceName,
329316
:name,
330317
:description,
331-
:jobContextUuid,
318+
null,
332319
:location,
333320
:inputs,
334321
:symlinkTargetId
@@ -344,7 +331,6 @@ JobRow upsertJob(
344331
String namespaceName,
345332
String name,
346333
String description,
347-
UUID jobContextUuid,
348334
String location,
349335
UUID symlinkTargetId,
350336
PGobject inputs);

0 commit comments

Comments
 (0)
⚡