Skip to content

Commit b64b7f9

Browse files
collado-mikecollado-mike
committed
Added parent job name to job models
Signed-off-by: Michael Collado <collado.mike@gmail.com>
1 parent 962d9cc commit b64b7f9

12 files changed

Lines changed: 33 additions & 9 deletions

File tree

‎api/src/main/java/marquez/db/Columns.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ private Columns() {}
8181
public static final String SCHEMA_LOCATION = "schema_location";
8282

8383
/* JOB ROW COLUMNS */
84+
public static final String PARENT_JOB_NAME = "parent_job_name";
8485
public static final String SIMPLE_NAME = "simple_name";
8586
public static final String SYMLINK_TARGET_UUID = "symlink_target_uuid";
8687

‎api/src/main/java/marquez/db/mappers/JobDataMapper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public JobData map(@NonNull ResultSet results, @NonNull StatementContext context
3838
JobType.valueOf(stringOrThrow(results, Columns.TYPE)),
3939
JobName.of(stringOrThrow(results, Columns.NAME)),
4040
stringOrThrow(results, Columns.SIMPLE_NAME),
41+
stringOrNull(results, Columns.PARENT_JOB_NAME),
4142
timestampOrThrow(results, Columns.CREATED_AT),
4243
timestampOrThrow(results, Columns.UPDATED_AT),
4344
NamespaceName.of(stringOrThrow(results, Columns.NAMESPACE_NAME)),

‎api/src/main/java/marquez/db/mappers/JobMapper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public Job map(@NonNull ResultSet results, @NonNull StatementContext context)
4545
JobType.valueOf(stringOrThrow(results, Columns.TYPE)),
4646
JobName.of(stringOrThrow(results, Columns.NAME)),
4747
stringOrThrow(results, Columns.SIMPLE_NAME),
48+
stringOrNull(results, Columns.PARENT_JOB_NAME),
4849
timestampOrThrow(results, Columns.CREATED_AT),
4950
timestampOrThrow(results, Columns.UPDATED_AT),
5051
getDatasetFromJsonOrNull(results, "current_inputs"),

‎api/src/main/java/marquez/db/mappers/JobRowMapper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public JobRow map(@NonNull ResultSet results, @NonNull StatementContext context)
4040
stringOrThrow(results, Columns.NAMESPACE_NAME),
4141
stringOrThrow(results, Columns.NAME),
4242
stringOrThrow(results, Columns.SIMPLE_NAME),
43+
stringOrNull(results, Columns.PARENT_JOB_NAME),
4344
stringOrNull(results, Columns.DESCRIPTION),
4445
uuidOrNull(results, Columns.CURRENT_VERSION_UUID),
4546
uuidOrNull(results, "current_job_context_uuid"),

‎api/src/main/java/marquez/db/models/JobData.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public class JobData implements NodeData {
3232
@NonNull JobType type;
3333
@NonNull JobName name;
3434
@NonNull String simpleName;
35+
@Nullable String parentJobName;
3536
@NonNull Instant createdAt;
3637
@NonNull Instant updatedAt;
3738
@NonNull NamespaceName namespace;

‎api/src/main/java/marquez/db/models/JobRow.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public class JobRow {
2020
@NonNull String namespaceName;
2121
@NonNull String name;
2222
@NonNull String simpleName;
23+
@Nullable String parentJobName;
2324
@Nullable String description;
2425
@Nullable UUID currentVersionUuid;
2526
@Nullable UUID jobContextUuid;

‎api/src/main/java/marquez/service/models/Job.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public final class Job {
2727
@Getter private final JobType type;
2828
@Getter private final JobName name;
2929
@Getter private final String simpleName;
30+
@Getter private final String parentJobName;
3031
@Getter private final Instant createdAt;
3132
@Getter private final Instant updatedAt;
3233
@Getter private final NamespaceName namespace;
@@ -44,6 +45,7 @@ public Job(
4445
@NonNull final JobType type,
4546
@NonNull final JobName name,
4647
@NonNull String simpleName,
48+
@Nullable String parentJobName,
4749
@NonNull final Instant createdAt,
4850
@NonNull final Instant updatedAt,
4951
@NonNull final Set<DatasetId> inputs,
@@ -58,6 +60,7 @@ public Job(
5860
this.type = type;
5961
this.name = name;
6062
this.simpleName = simpleName;
63+
this.parentJobName = parentJobName;
6164
this.createdAt = createdAt;
6265
this.updatedAt = updatedAt;
6366
this.namespace = id.getNamespace();

‎api/src/main/resources/marquez/db/migration/V43__alter_jobs_add_job_parent_uuid.sql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ CREATE OR REPLACE VIEW jobs_view
1414
AS
1515
with recursive
1616
job_fqn AS (
17-
SELECT uuid, name, namespace_name, parent_job_uuid
17+
SELECT uuid, name, namespace_name, NULL::text AS parent_job_name, parent_job_uuid
1818
FROM jobs
19-
UNION ALL
19+
UNION
2020
SELECT j1.uuid,
2121
CASE WHEN j2.name IS NOT NULL THEN j2.name || '.' || j1.name ELSE j1.name END AS name,
2222
CASE WHEN j2.namespace_name IS NOT NULL THEN j2.namespace_name ELSE j1.namespace_name END AS namespace_name,
23+
j2.name AS parent_job_name,
2324
j2.parent_job_uuid
2425
FROM jobs j1
2526
INNER JOIN job_fqn j2 ON j2.uuid=j1.parent_job_uuid
@@ -29,6 +30,7 @@ AS
2930
f.namespace_name,
3031
j.name AS simple_name,
3132
j.parent_job_uuid,
33+
f.parent_job_name,
3234
j.type,
3335
j.created_at,
3436
j.updated_at,

‎api/src/test/java/marquez/OpenLineageIntegrationTest.java‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -202,12 +202,14 @@ public void testOpenLineageJobHierarchyAirflowIntegration()
202202
Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name);
203203
assertThat(job)
204204
.isNotNull()
205-
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name));
205+
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name))
206+
.hasFieldOrPropertyWithValue("parentJobName", dagName);
206207

207208
Job parentJob = client.getJob(NAMESPACE_NAME, dagName);
208209
assertThat(parentJob)
209210
.isNotNull()
210-
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName));
211+
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName))
212+
.hasFieldOrPropertyWithValue("parentJobName", null);
211213
List<Run> runsList = client.listRuns(NAMESPACE_NAME, dagName);
212214
assertThat(runsList).isNotEmpty().hasSize(1);
213215
}
@@ -244,12 +246,14 @@ public void testOpenLineageJobHierarchyOldAirflowIntegration()
244246
assertThat(job)
245247
.isNotNull()
246248
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name))
247-
.hasFieldOrPropertyWithValue("simpleName", task1Name);
249+
.hasFieldOrPropertyWithValue("simpleName", task1Name)
250+
.hasFieldOrPropertyWithValue("parentJobName", dagName);
248251

249252
Job parentJob = client.getJob(NAMESPACE_NAME, dagName);
250253
assertThat(parentJob)
251254
.isNotNull()
252-
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName));
255+
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName))
256+
.hasFieldOrPropertyWithValue("parentJobName", null);
253257
List<Run> runsList = client.listRuns(NAMESPACE_NAME, dagName);
254258
assertThat(runsList).isNotEmpty().hasSize(1);
255259
UUID parentRunUuid = Utils.toNameBasedUuid(dagName, airflowParentRunId);
@@ -290,19 +294,22 @@ public void testOpenLineageJobHierarchySparkAndAirflow()
290294
assertThat(airflowTask)
291295
.isNotNull()
292296
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name))
293-
.hasFieldOrPropertyWithValue("simpleName", task1Name);
297+
.hasFieldOrPropertyWithValue("simpleName", task1Name)
298+
.hasFieldOrPropertyWithValue("parentJobName", dagName);
294299

295300
Job sparkJob = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name + "." + sparkTaskName);
296301
assertThat(sparkJob)
297302
.isNotNull()
298303
.hasFieldOrPropertyWithValue(
299304
"id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name + "." + sparkTaskName))
300-
.hasFieldOrPropertyWithValue("simpleName", sparkTaskName);
305+
.hasFieldOrPropertyWithValue("simpleName", sparkTaskName)
306+
.hasFieldOrPropertyWithValue("parentJobName", dagName + "." + task1Name);
301307

302308
Job parentJob = client.getJob(NAMESPACE_NAME, dagName);
303309
assertThat(parentJob)
304310
.isNotNull()
305-
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName));
311+
.hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName))
312+
.hasFieldOrPropertyWithValue("parentJobName", null);
306313
List<Run> runsList = client.listRuns(NAMESPACE_NAME, dagName);
307314
assertThat(runsList).isNotEmpty().hasSize(1);
308315
}

‎clients/java/src/main/java/marquez/client/models/Job.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public final class Job extends JobMeta {
2323
@Getter private final JobId id;
2424
@Getter private final String name;
2525
@Getter private final String simpleName;
26+
@Getter private final String parentJobName;
2627
@Getter private final Instant createdAt;
2728
@Getter private final Instant updatedAt;
2829
@Getter private final String namespace;
@@ -35,6 +36,7 @@ public Job(
3536
final JobType type,
3637
@NonNull final String name,
3738
@NonNull final String simpleName,
39+
final String parentJobName,
3840
@NonNull final Instant createdAt,
3941
@NonNull final Instant updatedAt,
4042
@NonNull final String namespace,
@@ -50,6 +52,7 @@ public Job(
5052
this.id = id;
5153
this.name = name;
5254
this.simpleName = simpleName;
55+
this.parentJobName = parentJobName;
5356
this.createdAt = createdAt;
5457
this.updatedAt = updatedAt;
5558
this.namespace = namespace;

0 commit comments

Comments
 (0)
⚡