Skip to content

Commit 87b8de3

Browse files
committed
Add timeout for resource allocation in lock step and freestyle jobs
Adds timeoutForAllocateResource and timeoutUnit parameters to the lock() pipeline step and lockTimeout/lockTimeoutUnit to freestyle job resource configuration. When a timeout is set and the resource is not acquired within the specified duration, the pipeline build fails with a LockWaitTimeoutException and the freestyle queue item is cancelled. Implementation uses a single scheduled task targeting the earliest deadline across all queued entries, with a periodic safety-net fallback. Fixes #866 Fixes #849 Fixes #30
1 parent 8ce5140 commit 87b8de3

20 files changed

Lines changed: 950 additions & 11 deletions
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
## Lock with allocation timeout
2+
3+
By default, the `lock` step waits indefinitely until the requested resource becomes available.
4+
With `timeoutForAllocateResource` you can set a maximum wait time — if the resource is not
5+
acquired within that period, the build fails immediately instead of blocking the queue forever.
6+
7+
This is useful when:
8+
- You prefer a fast failure over an indefinitely blocked pipeline
9+
- You want to detect resource starvation early
10+
- Your CI/CD has SLAs that cap how long a job may wait
11+
12+
### Pipeline (scripted)
13+
14+
```groovy
15+
node {
16+
// Wait up to 5 minutes for the resource, then fail
17+
lock(resource: 'my-printer', timeoutForAllocateResource: 5, timeoutUnit: 'MINUTES') {
18+
echo "Printer locked, printing ..."
19+
}
20+
}
21+
```
22+
23+
### Pipeline (declarative)
24+
25+
```groovy
26+
pipeline {
27+
agent any
28+
stages {
29+
stage('Deploy') {
30+
options {
31+
lock(resource: 'staging-env', timeoutForAllocateResource: 10, timeoutUnit: 'MINUTES')
32+
}
33+
steps {
34+
echo "Deploying to staging ..."
35+
}
36+
}
37+
}
38+
}
39+
```
40+
41+
### Label-based locking with timeout
42+
43+
```groovy
44+
pipeline {
45+
agent any
46+
stages {
47+
stage('Test') {
48+
steps {
49+
lock(label: 'phone', quantity: 1, variable: 'PHONE',
50+
timeoutForAllocateResource: 2, timeoutUnit: 'MINUTES') {
51+
echo "Running tests on ${env.PHONE}"
52+
}
53+
}
54+
}
55+
}
56+
}
57+
```
58+
59+
### Freestyle jobs
60+
61+
In a freestyle job configuration, go to **This build requires lockable resources** and set:
62+
- **Lock wait timeout**: the maximum time to wait (e.g. `5`)
63+
- **Timeout unit**: `SECONDS`, `MINUTES`, or `HOURS`
64+
65+
If the resource is not available within the configured timeout, the build is automatically
66+
removed from the Jenkins queue.
67+
68+
### Timeout values
69+
70+
| `timeoutUnit` | Description |
71+
|---------------|-------------|
72+
| `SECONDS` | Timeout in seconds |
73+
| `MINUTES` | Timeout in minutes (default) |
74+
| `HOURS` | Timeout in hours |
75+
76+
Setting `timeoutForAllocateResource: 0` (the default) disables the timeout — the build
77+
waits indefinitely, which preserves the original behaviour.

src/doc/examples/readme.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ If you have a question, please open a [GitHub issue](https://github.com/jenkinsc
1111
- [Locking a random free resource](locking-random-free-resource.md)
1212
- [Scripted vs declarative pipeline](scripted-vs-declarative-pipeline.md)
1313
- [Dynamic resource pool expansion](dynamic-resource-pool-expansion.md)
14+
- [Lock with allocation timeout](lock-with-timeout.md)

src/main/java/org/jenkins/plugins/lockableresources/LockStep.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,20 @@ public class LockStep extends Step implements Serializable {
6666
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
6767
public int priority = 0;
6868

69+
/**
70+
* Timeout in the specified {@link #timeoutUnit} for waiting to acquire the resource.
71+
* 0 means no timeout (wait indefinitely). When the timeout expires, the step fails
72+
* with an exception instead of waiting forever.
73+
*/
74+
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
75+
public long timeoutForAllocateResource = 0;
76+
77+
/**
78+
* Time unit for {@link #timeoutForAllocateResource}. Defaults to MINUTES.
79+
*/
80+
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
81+
public String timeoutUnit = "MINUTES";
82+
6983
// it should be LockStep() - without params. But keeping this for backward compatibility
7084
// so `lock('resource1')` still works and `lock(label: 'label1', quantity: 3)` works too (resource
7185
// is not required)
@@ -142,6 +156,24 @@ public void setExtra(@CheckForNull List<LockStepResource> extra) {
142156
this.extra = extra;
143157
}
144158

159+
@DataBoundSetter
160+
public void setTimeoutForAllocateResource(long timeoutForAllocateResource) {
161+
this.timeoutForAllocateResource = Math.max(0, timeoutForAllocateResource);
162+
}
163+
164+
@DataBoundSetter
165+
public void setTimeoutUnit(String timeoutUnit) {
166+
if (timeoutUnit != null && !timeoutUnit.trim().isEmpty()) {
167+
// Validate it is a valid TimeUnit name
168+
try {
169+
java.util.concurrent.TimeUnit.valueOf(timeoutUnit.toUpperCase(Locale.ENGLISH));
170+
} catch (IllegalArgumentException e) {
171+
throw new IllegalArgumentException("Invalid timeoutUnit: " + timeoutUnit);
172+
}
173+
this.timeoutUnit = timeoutUnit.toUpperCase(Locale.ENGLISH);
174+
}
175+
}
176+
145177
@Extension
146178
public static final class DescriptorImpl extends StepDescriptor {
147179

@@ -217,6 +249,20 @@ public static FormValidation doCheckResourceSelectStrategy(
217249
return FormValidation.ok();
218250
}
219251

252+
@RequirePOST
253+
public ListBoxModel doFillTimeoutUnitItems(@AncestorInPath Item item) {
254+
if (item != null) {
255+
item.checkPermission(Item.CONFIGURE);
256+
} else {
257+
Jenkins.get().checkPermission(Jenkins.ADMINISTER);
258+
}
259+
ListBoxModel items = new ListBoxModel();
260+
items.add("Seconds", "SECONDS");
261+
items.add("Minutes", "MINUTES");
262+
items.add("Hours", "HOURS");
263+
return items;
264+
}
265+
220266
@Override
221267
public Set<Class<?>> getRequiredContext() {
222268
return Collections.singleton(TaskListener.class);

src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,16 +147,22 @@ private void onLockFailed(PrintStream logger, List<LockableResourcesStruct> reso
147147
getContext().onSuccess(null);
148148
} else {
149149
this.printBlockCause(logger, resourceHolderList);
150-
LockableResourcesManager.printLogs(
151-
"[" + step + "] is not free, waiting for execution ...", Level.FINE, LOGGER, logger);
150+
String waitMsg = "[" + step + "] is not free, waiting for execution ...";
151+
if (step.timeoutForAllocateResource > 0) {
152+
waitMsg += " (timeout: " + step.timeoutForAllocateResource + " "
153+
+ step.timeoutUnit.toLowerCase(java.util.Locale.ENGLISH) + ")";
154+
}
155+
LockableResourcesManager.printLogs(waitMsg, Level.FINE, LOGGER, logger);
152156
LockableResourcesManager lrm = LockableResourcesManager.get();
153157
lrm.queueContext(
154158
getContext(),
155159
resourceHolderList,
156160
step.toString(),
157161
step.variable,
158162
step.inversePrecedence,
159-
step.priority);
163+
step.priority,
164+
step.timeoutForAllocateResource,
165+
step.timeoutUnit);
160166
}
161167
}
162168

src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java

Lines changed: 133 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,12 @@ public class LockableResourcesManager extends GlobalConfiguration {
9595
private transient volatile AtomicBoolean savePending;
9696
private transient volatile ScheduledExecutorService saveExecutor;
9797

98+
/** Single scheduled timeout task. Guarded by {@link #syncResources}. */
99+
private transient java.util.concurrent.ScheduledFuture<?> nextTimeoutTask;
100+
101+
/** Deadline (epoch ms) the current {@link #nextTimeoutTask} targets. 0 = none. */
102+
private transient long nextTimeoutDeadline;
103+
98104
@DataBoundSetter
99105
public void setAllowEmptyOrNullValues(boolean allowEmptyOrNullValues) {
100106
this.allowEmptyOrNullValues = allowEmptyOrNullValues;
@@ -853,8 +859,9 @@ private QueuedContextStruct getNextQueuedContext() {
853859

854860
LOGGER.fine("current queue size: " + this.queuedContexts.size());
855861
LOGGER.finest("current queue: " + this.queuedContexts);
856-
List<QueuedContextStruct> orphan = new ArrayList<>();
862+
List<QueuedContextStruct> toRemove = new ArrayList<>();
857863
QueuedContextStruct nextEntry = null;
864+
long earliestDeadline = Long.MAX_VALUE;
858865

859866
// the first one added lock is the oldest one, and this wins
860867

@@ -863,18 +870,43 @@ private QueuedContextStruct getNextQueuedContext() {
863870
// check queue list first
864871
if (!entry.isValid()) {
865872
LOGGER.fine("well be removed: " + idx + " " + entry);
866-
orphan.add(entry);
873+
toRemove.add(entry);
874+
continue;
875+
}
876+
877+
// check if the entry has timed out waiting for resources
878+
if (entry.isTimedOut()) {
879+
LOGGER.info("Queue entry timed out waiting for resources: " + entry);
880+
toRemove.add(entry);
881+
PrintStream logger = entry.getLogger();
882+
String msg = "[" + entry.getResourceDescription()
883+
+ "] timed out waiting for resource allocation after "
884+
+ entry.getTimeoutForAllocateResource() + " "
885+
+ entry.getTimeoutUnit().toLowerCase(java.util.Locale.ENGLISH);
886+
printLogs(msg, logger, Level.WARNING);
887+
entry.getContext()
888+
.onFailure(new org.jenkins.plugins.lockableresources.queue.LockWaitTimeoutException(msg));
867889
continue;
868890
}
891+
892+
// track the earliest deadline among remaining entries for rescheduling
893+
long deadline = entry.getTimeoutDeadlineMillis();
894+
if (deadline > 0 && deadline < earliestDeadline) {
895+
earliestDeadline = deadline;
896+
}
897+
869898
LOGGER.finest("oldest win - index: " + idx + " " + entry);
870899

871900
nextEntry = getNextQueuedContextEntry(entry);
872901
}
873902

874-
if (!orphan.isEmpty()) {
875-
this.queuedContexts.removeAll(orphan);
903+
if (!toRemove.isEmpty()) {
904+
this.queuedContexts.removeAll(toRemove);
876905
}
877906

907+
// reschedule for the next earliest deadline
908+
scheduleTimeoutAt(earliestDeadline);
909+
878910
return nextEntry;
879911
}
880912

@@ -1407,6 +1439,32 @@ public void queueContext(
14071439
String variableName,
14081440
boolean inversePrecedence,
14091441
int priority) {
1442+
queueContext(
1443+
context,
1444+
requiredResources,
1445+
resourceDescription,
1446+
variableName,
1447+
inversePrecedence,
1448+
priority,
1449+
0,
1450+
"MINUTES");
1451+
}
1452+
1453+
// ---------------------------------------------------------------------------
1454+
/*
1455+
* Adds the given context and the required resources to the queue if
1456+
* this context is not yet queued, with a timeout for resource allocation.
1457+
*/
1458+
@Restricted(NoExternalUse.class)
1459+
public void queueContext(
1460+
StepContext context,
1461+
List<LockableResourcesStruct> requiredResources,
1462+
String resourceDescription,
1463+
String variableName,
1464+
boolean inversePrecedence,
1465+
int priority,
1466+
long timeoutForAllocateResource,
1467+
String timeoutUnit) {
14101468
synchronized (syncResources) {
14111469
for (QueuedContextStruct entry : this.queuedContexts) {
14121470
if (entry.getContext() == context) {
@@ -1416,8 +1474,14 @@ public void queueContext(
14161474
}
14171475

14181476
int queueIndex = 0;
1419-
QueuedContextStruct newQueueItem =
1420-
new QueuedContextStruct(context, requiredResources, resourceDescription, variableName, priority);
1477+
QueuedContextStruct newQueueItem = new QueuedContextStruct(
1478+
context,
1479+
requiredResources,
1480+
resourceDescription,
1481+
variableName,
1482+
priority,
1483+
timeoutForAllocateResource,
1484+
timeoutUnit);
14211485

14221486
if (!inversePrecedence || priority != 0) {
14231487
queueIndex = this.queuedContexts.size() - 1;
@@ -1439,6 +1503,13 @@ public void queueContext(
14391503
Level.FINE);
14401504

14411505
save();
1506+
1507+
// If this entry has a timeout and its deadline is earlier than the
1508+
// currently scheduled one, (re)schedule so it fires on time.
1509+
long deadline = newQueueItem.getTimeoutDeadlineMillis();
1510+
if (deadline > 0 && (nextTimeoutDeadline == 0 || deadline < nextTimeoutDeadline)) {
1511+
scheduleTimeoutAt(deadline);
1512+
}
14421513
}
14431514
}
14441515

@@ -1495,7 +1566,7 @@ public void refreshQueue() {
14951566
// Invalidate cached candidates so waiting jobs re-evaluate with current labels
14961567
cachedCandidates.invalidateAll();
14971568

1498-
// Process waiting pipeline jobs
1569+
// Process waiting pipeline jobs (also handles timeouts)
14991570
synchronized (syncResources) {
15001571
while (proceedNextContext()) {
15011572
// process as many contexts as possible
@@ -1506,6 +1577,61 @@ public void refreshQueue() {
15061577
scheduleQueueMaintenance();
15071578
}
15081579

1580+
// ---------------------------------------------------------------------------
1581+
/**
1582+
* Checks for timed-out entries in the pipeline lock queue and fails them.
1583+
* Called by {@link org.jenkins.plugins.lockableresources.queue.LockWaitTimeoutPeriodicWork}
1584+
* as a safety net.
1585+
*/
1586+
@Restricted(NoExternalUse.class)
1587+
public void checkTimeouts() {
1588+
synchronized (syncResources) {
1589+
// proceedNextContext → getNextQueuedContext handles timeouts + rescheduling
1590+
while (proceedNextContext()) {
1591+
// process as many contexts as possible
1592+
}
1593+
}
1594+
}
1595+
1596+
// ---------------------------------------------------------------------------
1597+
/**
1598+
* Schedules (or reschedules) the single timeout task to fire at the given
1599+
* deadline. If {@code deadline} is {@link Long#MAX_VALUE} the current task
1600+
* is cancelled and nothing new is scheduled.
1601+
* Must be called while holding {@link #syncResources}.
1602+
*/
1603+
private void scheduleTimeoutAt(long deadline) {
1604+
// Cancel the current task — we will either replace it or clear it
1605+
if (nextTimeoutTask != null) {
1606+
nextTimeoutTask.cancel(false);
1607+
nextTimeoutTask = null;
1608+
nextTimeoutDeadline = 0;
1609+
}
1610+
1611+
if (deadline == Long.MAX_VALUE || deadline <= 0) {
1612+
return;
1613+
}
1614+
1615+
nextTimeoutDeadline = deadline;
1616+
// Small buffer so the deadline has definitely passed when we check
1617+
long delayMs = Math.max(0, deadline - System.currentTimeMillis()) + 500L;
1618+
LOGGER.fine("Scheduling timeout check in " + delayMs + "ms");
1619+
nextTimeoutTask = jenkins.util.Timer.get()
1620+
.schedule(
1621+
() -> {
1622+
LOGGER.fine("Scheduled timeout check fired");
1623+
synchronized (syncResources) {
1624+
nextTimeoutDeadline = 0;
1625+
nextTimeoutTask = null;
1626+
while (proceedNextContext()) {
1627+
// process as many contexts as possible
1628+
}
1629+
}
1630+
},
1631+
delayMs,
1632+
java.util.concurrent.TimeUnit.MILLISECONDS);
1633+
}
1634+
15091635
// ---------------------------------------------------------------------------
15101636
private AtomicBoolean getSavePending() {
15111637
AtomicBoolean sp = savePending;

0 commit comments

Comments
 (0)