Skip to content

Commit 95c00c8

Browse files
JDarDagranJDarDagran
committed
Remove job context.
Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Add call to API in frontend. Add tests. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Update javadocs. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Apply spotless fixes. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Remove FacetResource. Split react functions to each component. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Add more tests for JobResource. Update OpenAPI spec. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Add more tests. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Add lombok option to skip coverage check on NonNull. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com> Move find facets methods to corresponding daos. Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com>
1 parent 9f86e50 commit 95c00c8

90 files changed

Lines changed: 921 additions & 504 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/java/marquez/MarquezContext.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,15 @@
2828
import marquez.db.DatasetDao;
2929
import marquez.db.DatasetFieldDao;
3030
import marquez.db.DatasetVersionDao;
31-
import marquez.db.JobContextDao;
3231
import marquez.db.JobDao;
32+
import marquez.db.JobFacetsDao;
3333
import marquez.db.JobVersionDao;
3434
import marquez.db.LineageDao;
3535
import marquez.db.NamespaceDao;
3636
import marquez.db.OpenLineageDao;
3737
import marquez.db.RunArgsDao;
3838
import marquez.db.RunDao;
39+
import marquez.db.RunFacetsDao;
3940
import marquez.db.RunStateDao;
4041
import marquez.db.SearchDao;
4142
import marquez.db.SourceDao;
@@ -67,9 +68,10 @@ public final class MarquezContext {
6768
@Getter private final DatasetVersionDao datasetVersionDao;
6869
@Getter private final JobDao jobDao;
6970
@Getter private final JobVersionDao jobVersionDao;
70-
@Getter private final JobContextDao jobContextDao;
71+
@Getter private final JobFacetsDao jobFacetsDao;
7172
@Getter private final RunDao runDao;
7273
@Getter private final RunArgsDao runArgsDao;
74+
@Getter private final RunFacetsDao runFacetsDao;
7375
@Getter private final RunStateDao runStateDao;
7476
@Getter private final TagDao tagDao;
7577
@Getter private final OpenLineageDao openLineageDao;
@@ -116,9 +118,10 @@ private MarquezContext(
116118
this.datasetVersionDao = jdbi.onDemand(DatasetVersionDao.class);
117119
this.jobDao = jdbi.onDemand(JobDao.class);
118120
this.jobVersionDao = jdbi.onDemand(JobVersionDao.class);
119-
this.jobContextDao = jdbi.onDemand(JobContextDao.class);
121+
this.jobFacetsDao = jdbi.onDemand(JobFacetsDao.class);
120122
this.runDao = jdbi.onDemand(RunDao.class);
121123
this.runArgsDao = jdbi.onDemand(RunArgsDao.class);
124+
this.runFacetsDao = jdbi.onDemand(RunFacetsDao.class);
122125
this.runStateDao = jdbi.onDemand(RunStateDao.class);
123126
this.tagDao = jdbi.onDemand(TagDao.class);
124127
this.openLineageDao = jdbi.onDemand(OpenLineageDao.class);
@@ -158,7 +161,7 @@ private MarquezContext(
158161
this.sourceResource = new SourceResource(serviceFactory);
159162
this.datasetResource = new DatasetResource(serviceFactory);
160163
this.columnLineageResource = new ColumnLineageResource(serviceFactory);
161-
this.jobResource = new JobResource(serviceFactory, jobVersionDao);
164+
this.jobResource = new JobResource(serviceFactory, jobVersionDao, jobFacetsDao, runFacetsDao);
162165
this.tagResource = new TagResource(serviceFactory);
163166
this.openLineageResource = new OpenLineageResource(serviceFactory, openLineageDao);
164167
this.searchResource = new SearchResource(searchDao);

‎api/src/main/java/marquez/api/JobResource.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import java.util.List;
1616
import javax.validation.Valid;
1717
import javax.validation.constraints.Min;
18+
import javax.validation.constraints.NotNull;
1819
import javax.ws.rs.Consumes;
1920
import javax.ws.rs.DELETE;
2021
import javax.ws.rs.DefaultValue;
@@ -37,11 +38,14 @@
3738
import marquez.api.exceptions.JobVersionNotFoundException;
3839
import marquez.api.models.JobVersion;
3940
import marquez.api.models.ResultsPage;
41+
import marquez.common.models.FacetType;
4042
import marquez.common.models.JobName;
4143
import marquez.common.models.NamespaceName;
4244
import marquez.common.models.RunId;
4345
import marquez.common.models.Version;
46+
import marquez.db.JobFacetsDao;
4447
import marquez.db.JobVersionDao;
48+
import marquez.db.RunFacetsDao;
4549
import marquez.db.models.JobRow;
4650
import marquez.service.ServiceFactory;
4751
import marquez.service.models.Job;
@@ -52,11 +56,18 @@
5256
@Path("/api/v1")
5357
public class JobResource extends BaseResource {
5458
private final JobVersionDao jobVersionDao;
59+
private final JobFacetsDao jobFacetsDao;
60+
private final RunFacetsDao runFacetsDao;
5561

5662
public JobResource(
57-
@NonNull final ServiceFactory serviceFactory, @NonNull final JobVersionDao jobVersionDao) {
63+
@NonNull final ServiceFactory serviceFactory,
64+
@NonNull final JobVersionDao jobVersionDao,
65+
@NonNull JobFacetsDao jobFacetsDao,
66+
@NonNull RunFacetsDao runFacetsDao) {
5867
super(serviceFactory);
5968
this.jobVersionDao = jobVersionDao;
69+
this.jobFacetsDao = jobFacetsDao;
70+
this.runFacetsDao = runFacetsDao;
6071
}
6172

6273
/**
@@ -236,6 +247,33 @@ public RunResource runResourceRoot(@PathParam("id") RunId runId) {
236247
return new RunResource(runId, runService);
237248
}
238249

250+
@Timed
251+
@ResponseMetered
252+
@ExceptionMetered
253+
@GET
254+
@Produces(APPLICATION_JSON)
255+
@Path("/jobs/runs/{id}/facets")
256+
public Response getRunFacets(
257+
@PathParam("id") RunId runId, @QueryParam("type") @NotNull FacetType type) {
258+
throwIfNotExists(runId);
259+
Object facets = null;
260+
switch (type) {
261+
case JOB:
262+
facets = jobFacetsDao.findJobFacetsByRunUuid(runId.getValue());
263+
break;
264+
case RUN:
265+
facets = runFacetsDao.findRunFacetsByRunUuid(runId.getValue());
266+
break;
267+
case DATASET:
268+
// for future case if there's a need to add dataset facets to the endpoint
269+
break;
270+
default:
271+
break;
272+
}
273+
274+
return Response.ok(facets).build();
275+
}
276+
239277
@Value
240278
static class JobVersions {
241279
@NonNull

‎api/src/main/java/marquez/api/models/JobVersion.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
package marquez.api.models;
77

8-
import com.google.common.collect.ImmutableMap;
98
import java.net.URL;
109
import java.time.Instant;
1110
import java.util.List;
@@ -35,7 +34,6 @@ public final class JobVersion {
3534
@Getter private final Version version;
3635
@Getter private final NamespaceName namespace;
3736
@Nullable private final URL location;
38-
@Getter private final ImmutableMap<String, String> context;
3937
@Getter private final List<DatasetId> inputs;
4038
@Getter private final List<DatasetId> outputs;
4139
@Getter @Nullable private final Run latestRun;
@@ -46,7 +44,6 @@ public JobVersion(
4644
@NonNull final Instant createdAt,
4745
@NonNull final Version version,
4846
@Nullable final URL location,
49-
@Nullable final ImmutableMap<String, String> context,
5047
List<DatasetId> inputs,
5148
List<DatasetId> outputs,
5249
@Nullable Run latestRun) {
@@ -56,7 +53,6 @@ public JobVersion(
5653
this.version = version;
5754
this.namespace = id.getNamespace();
5855
this.location = location;
59-
this.context = (context == null) ? ImmutableMap.of() : context;
6056
this.inputs = inputs;
6157
this.outputs = outputs;
6258
this.latestRun = latestRun;

‎api/src/main/java/marquez/common/Utils.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.fasterxml.jackson.databind.SerializationFeature;
2121
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2222
import com.google.common.base.Joiner;
23-
import com.google.common.collect.ImmutableMap;
2423
import com.google.common.collect.ImmutableSet;
2524
import com.google.common.hash.Hashing;
2625
import io.dropwizard.jackson.Jackson;
@@ -236,7 +235,6 @@ public static Instant toInstant(@Nullable final String asIso) {
236235
* @param jobName The name of the job.
237236
* @param jobInputIds The input dataset IDs for the job.
238237
* @param jobOutputIds The output dataset IDs for the job.
239-
* @param jobContext The context of the job.
240238
* @param jobLocation The source code location for the job.
241239
* @return A {@link Version} object based on the specified job meta.
242240
*/
@@ -245,7 +243,6 @@ public static Version newJobVersionFor(
245243
@NonNull final JobName jobName,
246244
@NonNull final ImmutableSet<DatasetId> jobInputIds,
247245
@NonNull final ImmutableSet<DatasetId> jobOutputIds,
248-
@NonNull final ImmutableMap<String, String> jobContext,
249246
@Nullable final String jobLocation) {
250247
final byte[] bytes =
251248
VERSION_JOINER
@@ -268,8 +265,7 @@ public static Version newJobVersionFor(
268265
jobOutputId.getNamespace().getValue(),
269266
jobOutputId.getName().getValue()))
270267
.collect(joining(VERSION_DELIM)),
271-
jobLocation,
272-
KV_JOINER.join(jobContext))
268+
jobLocation)
273269
.getBytes(UTF_8);
274270
return Version.of(UUID.nameUUIDFromBytes(bytes));
275271
}

‎api/src/main/java/marquez/common/models/DatasetId.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import com.fasterxml.jackson.annotation.JsonProperty;
99
import com.google.common.collect.ComparisonChain;
10-
import com.google.common.collect.ImmutableMap;
1110
import com.google.common.collect.ImmutableSet;
1211
import lombok.EqualsAndHashCode;
1312
import lombok.NonNull;
@@ -16,8 +15,8 @@
1615

1716
/**
1817
* ID for {@code Dataset}. The class implements {@link Comparable} to ensure job versions generated
19-
* with {@link Utils#newJobVersionFor(NamespaceName, JobName, ImmutableSet, ImmutableSet,
20-
* ImmutableMap, String)} are consistent as jobs may contain inputs and outputs out of order.
18+
* with {@link Utils#newJobVersionFor(NamespaceName, JobName, ImmutableSet, ImmutableSet, String)}
19+
* are consistent as jobs may contain inputs and outputs out of order.
2120
*/
2221
@EqualsAndHashCode
2322
@ToString

‎api/src/main/java/marquez/common/models/FacetType.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/*
2+
* Copyright 2018-2023 contributors to the Marquez project
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package marquez.common.models;
7+
8+
public enum FacetType {
9+
RUN,
10+
JOB,
11+
DATASET;
12+
}

‎api/src/main/java/marquez/db/BaseDao.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ public interface BaseDao extends SqlObject {
1818
@CreateSqlObject
1919
DatasetVersionDao createDatasetVersionDao();
2020

21-
@CreateSqlObject
22-
JobContextDao createJobContextDao();
23-
2421
@CreateSqlObject
2522
JobDao createJobDao();
2623

‎api/src/main/java/marquez/db/Columns.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,9 @@ private Columns() {}
103103

104104
/* JOB VERSION ROW COLUMNS */
105105
public static final String JOB_UUID = "job_uuid";
106-
public static final String JOB_CONTEXT_UUID = "job_context_uuid";
107106
public static final String LOCATION = "location";
108107
public static final String LATEST_RUN_UUID = "latest_run_uuid";
109108

110-
/* JOB CONTEXT ROW COLUMNS */
111-
public static final String CONTEXT = "context";
112-
113109
/* RUN ROW COLUMNS */
114110
public static final String EXTERNAL_ID = "external_id";
115111
public static final String RUN_ARGS_UUID = "run_args_uuid";

‎api/src/main/java/marquez/db/JobContextDao.java‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎api/src/main/java/marquez/db/JobDao.java‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,13 @@
1515
import java.util.Set;
1616
import java.util.UUID;
1717
import java.util.stream.Collectors;
18-
import marquez.common.Utils;
1918
import marquez.common.models.DatasetId;
2019
import marquez.common.models.DatasetName;
2120
import marquez.common.models.JobName;
2221
import marquez.common.models.JobType;
2322
import marquez.common.models.NamespaceName;
2423
import marquez.db.mappers.JobMapper;
2524
import marquez.db.mappers.JobRowMapper;
26-
import marquez.db.models.JobContextRow;
2725
import marquez.db.models.JobRow;
2826
import marquez.db.models.NamespaceRow;
2927
import marquez.service.models.Job;
@@ -58,10 +56,9 @@ SELECT EXISTS (
5856

5957
@SqlQuery(
6058
"""
61-
SELECT j.*, jc.context, f.facets
59+
SELECT j.*, f.facets
6260
FROM jobs_view j
6361
LEFT OUTER JOIN job_versions AS jv ON jv.uuid = j.current_version_uuid
64-
LEFT OUTER JOIN job_contexts jc ON jc.uuid = j.current_job_context_uuid
6562
LEFT OUTER JOIN (
6663
SELECT run_uuid, JSON_AGG(e.facet) AS facets
6764
FROM (
@@ -130,10 +127,9 @@ default Optional<Job> findWithRun(String namespaceName, String jobName) {
130127

131128
@SqlQuery(
132129
"""
133-
SELECT j.*, jc.context, f.facets
130+
SELECT j.*, f.facets
134131
FROM jobs_view AS j
135132
LEFT OUTER JOIN job_versions AS jv ON jv.uuid = j.current_version_uuid
136-
LEFT OUTER JOIN job_contexts jc ON jc.uuid = j.current_job_context_uuid
137133
LEFT OUTER JOIN (
138134
SELECT run_uuid, JSON_AGG(e.facet) AS facets
139135
FROM (
@@ -208,13 +204,6 @@ default JobRow upsertJobMeta(
208204
createNamespaceDao()
209205
.upsertNamespaceRow(
210206
UUID.randomUUID(), createdAt, namespaceName.getValue(), DEFAULT_NAMESPACE_OWNER);
211-
JobContextRow contextRow =
212-
createJobContextDao()
213-
.upsert(
214-
UUID.randomUUID(),
215-
createdAt,
216-
Utils.toJson(jobMeta.getContext()),
217-
Utils.checksumFor(jobMeta.getContext()));
218207
return upsertJob(
219208
UUID.randomUUID(),
220209
jobMeta.getType(),
@@ -223,7 +212,6 @@ default JobRow upsertJobMeta(
223212
namespace.getName(),
224213
jobName.getValue(),
225214
jobMeta.getDescription().orElse(null),
226-
contextRow.getUuid(),
227215
toUrlString(jobMeta.getLocation().orElse(null)),
228216
symlinkTargetUuid,
229217
toJson(jobMeta.getInputs(), mapper));
@@ -276,7 +264,7 @@ INSERT INTO jobs_view AS j (
276264
:namespaceName,
277265
:name,
278266
:description,
279-
:jobContextUuid,
267+
null,
280268
:location,
281269
:inputs,
282270
:symlinkTargetId,
@@ -291,7 +279,6 @@ JobRow upsertJob(
291279
String namespaceName,
292280
String name,
293281
String description,
294-
UUID jobContextUuid,
295282
String location,
296283
UUID symlinkTargetId,
297284
PGobject inputs);
@@ -328,7 +315,7 @@ INSERT INTO jobs_view AS j (
328315
:namespaceName,
329316
:name,
330317
:description,
331-
:jobContextUuid,
318+
null,
332319
:location,
333320
:inputs,
334321
:symlinkTargetId
@@ -344,7 +331,6 @@ JobRow upsertJob(
344331
String namespaceName,
345332
String name,
346333
String description,
347-
UUID jobContextUuid,
348334
String location,
349335
UUID symlinkTargetId,
350336
PGobject inputs);

0 commit comments

Comments
 (0)
⚡