File D:\maven-qcr-plugin-testsunitaires\qcr4java\jenkins-master\target\site\qc\core\src\main\java\hudson\slaves\NodeProvisioner.java.html

1 /*
2 * The MIT License
3 *
4 * Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24 package hudson.slaves;
25
26 import hudson.model.*;
27 import jenkins.model.Jenkins;
28
29 import static hudson.model.LoadStatistics.DECAY;
30 import hudson.model.MultiStageTimeSeries.TimeScale;
31 import hudson.Extension;
32
33 import java.awt.Color;
34 import java.util.concurrent.Future;
35 import java.util.concurrent.ExecutionException;
36 import java.util.List;
37 import java.util.Collection;
38 import java.util.ArrayList;
39 import java.util.Iterator;
40 import java.util.concurrent.TimeUnit;
41 import java.util.logging.Logger;
42 import java.util.logging.Level;
43 import java.io.IOException;
44
45 /**
46 * Uses the {@link LoadStatistics} and determines when we need to allocate
47 * new {@link Node}s through {@link Cloud}.
48 *
49 * @author Kohsuke Kawaguchi
50 */
51 public class NodeProvisioner {
52 /**
53 * The node addition activity in progress.
54 */
55 public static class PlannedNode {
56 /**
57 * Used to display this planned node to UI. Should ideally include the identifier unique to the node
58 * being provisioned (like the instance ID), but if such an identifier doesn't readily exist, this
59 * can be just a name of the template being provisioned (like the machine image ID.)
60 */
61 public final String displayName;
62
63 /**
64 * Used to launch and return a {@link Node} object. {@link NodeProvisioner} will check
65 * this {@link Future}'s isDone() method to determine when to finalize this object.
66 */
67 public final Future<Node> future;
68
69 /**
70 * The number of executors that will be provided by the {@link Node} launched by
71 * this object. This is used for capacity planning in {@link NodeProvisioner#update}.
72 */
73 public final int numExecutors;
74
75 /**
76 * Construct a PlannedNode instance without {@link Cloud} callback for finalization.
77 *
78 * @param displayName Used to display this object in the UI.
79 * @param future Used to launch a @{link Node} object.
80 * @param numExecutors The number of executors that will be provided by the launched {@link Node}.
81 */
82 public PlannedNode(String displayName, Future<Node> future, int numExecutors) {
83 if(displayName==null || future==null || numExecutors<1) throw new IllegalArgumentException();
84 this.displayName = displayName;
85 this.future = future;
86 this.numExecutors = numExecutors;
87 }
88
89 /**
90 * Indicate that this {@link PlannedNode} is being finalized.
91 *
92 * <p>
93 * {@link NodeProvisioner} will call this method when it's done with {@link PlannedNode}.
94 * This indicates that the {@link PlannedNode}'s work has been completed
95 * (successfully or otherwise) and it is about to be removed from the list of pending
96 * {@link Node}s to be launched.
97 *
98 * <p>
99 * Create a subtype of this class and override this method to add any necessary behaviour.
100 *
101 * @since 1.503
102 */
103 public void spent() {
104 }
105 }
106
107 /**
108 * Load for the label.
109 */
110 private final LoadStatistics stat;
111
112 /**
113 * For which label are we working?
114 * Null if this {@link NodeProvisioner} is working for the entire Hudson,
115 * for jobs that are unassigned to any particular node.
116 */
117 private final Label label;
118
119 private List<PlannedNode> pendingLaunches = new ArrayList<PlannedNode>();
120
121 private transient volatile long lastSuggestedReview;
122
123 /**
124 * Exponential moving average of the "planned capacity" over time, which is the number of
125 * additional executors being brought up.
126 *
127 * This is used to filter out high-frequency components from the planned capacity, so that
128 * the comparison with other low-frequency only variables won't leave spikes.
129 */
130 private final MultiStageTimeSeries plannedCapacitiesEMA =
131 new MultiStageTimeSeries(Messages._NodeProvisioner_EmptyString(),Color.WHITE,0,DECAY);
132
133 public NodeProvisioner(Label label, LoadStatistics loadStatistics) {
134 this.label = label;
135 this.stat = loadStatistics;
136 }
137
138 /**
139 * Nodes that are being launched.
140 *
141 * @return
142 * Can be empty but never null
143 * @since 1.401
144 */
145 public synchronized List<PlannedNode> getPendingLaunches() {
146 return new ArrayList<PlannedNode>(pendingLaunches);
147 }
148
149 /**
150 * Give the {@link NodeProvisioner} a hint that now would be a good time to think about provisioning some nodes.
151 * The hint will be ignored if subjected to excessive pestering by callers.
152 *
153 * @since 1.415
154 */
155 public void suggestReviewNow() {
156 if (System.currentTimeMillis() > lastSuggestedReview + TimeUnit.SECONDS.toMillis(1)) {
157 lastSuggestedReview = System.currentTimeMillis();
158 Computer.threadPoolForRemoting.submit(new Runnable() {
159 public void run() {
160 update();
161 }
162 });
163 }
164 }
165
166 /**
167 * Periodically invoked to keep track of the load.
168 * Launches additional nodes if necessary.
169 */
170 private synchronized void update() {
171 Jenkins hudson = Jenkins.getInstance();
172 lastSuggestedReview = System.currentTimeMillis();
173
174 // clean up the cancelled launch activity, then count the # of executors that we are about to bring up.
175 int plannedCapacitySnapshot = 0;
176 for (Iterator<PlannedNode> itr = pendingLaunches.iterator(); itr.hasNext();) {
177 PlannedNode f = itr.next();
178 if(f.future.isDone()) {
179 try {
180 hudson.addNode(f.future.get());
181 LOGGER.info(f.displayName+" provisioning successfully completed. We have now "+hudson.getComputers().length+" computer(s)");
182 } catch (InterruptedException e) {
183 throw new AssertionError(e); // since we confirmed that the future is already done
184 } catch (ExecutionException e) {
185 LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e.getCause());
186 } catch (IOException e) {
187 LOGGER.log(Level.WARNING, "Provisioned slave "+f.displayName+" failed to launch",e);
188 }
189
190 f.spent();
191
192 itr.remove();
193 } else
194 plannedCapacitySnapshot += f.numExecutors;
195 }
196 float plannedCapacity = plannedCapacitySnapshot;
197 plannedCapacitiesEMA.update(plannedCapacity);
198
199 /*
200 Here we determine how many additional slaves we need to keep up with the load (if at all),
201 which involves a simple math.
202
203 Broadly speaking, first we check that all the executors are fully utilized before attempting
204 to start any new slave (this also helps to ignore the temporary gap between different numbers,
205 as changes in them are not necessarily synchronized --- for example, there's a time lag between
206 when a slave launches (thus bringing the planned capacity down) and the time when its executors
207 pick up builds (thus bringing the queue length down.)
208
209 Once we confirm that, we compare the # of buildable items against the additional slaves
210 that are being brought online. If we have more jobs than our executors can handle, we'll launch a new slave.
211
212 So this computation involves three stats:
213
214 1. # of idle executors
215 2. # of jobs that are starving for executors
216 3. # of additional slaves being provisioned (planned capacities.)
217
218 To ignore a temporary surge/drop, we make conservative estimates on each one of them. That is,
219 we take the current snapshot value, and we take the current exponential moving average (EMA) value,
220 and use the max/min.
221
222 This is another measure to be robust against temporary surge/drop in those indicators, and helps
223 us avoid over-reacting to stats.
224
225 If we only use the snapshot value or EMA value, tests confirmed that the gap creates phantom
226 excessive loads and Hudson ends up firing excessive capacities. In a static system, over the time
227 EMA and the snapshot value becomes the same, so this makes sure that in a long run this conservative
228 estimate won't create a starvation.
229 */
230
231 int idleSnapshot = stat.computeIdleExecutors();
232 int totalSnapshot = stat.computeTotalExecutors();
233 boolean needSomeWhenNoneAtAll = (idleSnapshot==0) && (totalSnapshot + plannedCapacitySnapshot == 0) && (stat.computeQueueLength() > 0);
234 float idle = Math.max(stat.getLatestIdleExecutors(TIME_SCALE), idleSnapshot);
235 if(idle<MARGIN || needSomeWhenNoneAtAll) {
236 // make sure the system is fully utilized before attempting any new launch.
237
238 // this is the amount of work left to be done
239 float qlen = Math.min(stat.queueLength.getLatest(TIME_SCALE), stat.computeQueueLength());
240
241 // ... and this is the additional executors we've already provisioned.
242 plannedCapacity = Math.max(plannedCapacitiesEMA.getLatest(TIME_SCALE),plannedCapacity);
243
244 float excessWorkload = qlen - plannedCapacity;
245 if (needSomeWhenNoneAtAll && excessWorkload < 1) {
246 // in this specific exceptional case we should just provision right now
247 // the exponential smoothing will delay the build unnecessarily
248 excessWorkload = 1;
249 }
250 float m = calcThresholdMargin(totalSnapshot);
251 if(excessWorkload>1-m) {// and there's more work to do...
252 LOGGER.fine("Excess workload "+excessWorkload+" detected. (planned capacity="+plannedCapacity+",Qlen="+qlen+",idle="+idle+"&"+idleSnapshot+",total="+totalSnapshot+"m,="+m+")");
253 for( Cloud c : hudson.clouds ) {
254 if(excessWorkload<0) break; // enough slaves allocated
255
256 // Make sure this cloud actually can provision for this label.
257 if (c.canProvision(label)) {
258 // provisioning a new node should be conservative --- for example if exceeWorkload is 1.4,
259 // we don't want to allocate two nodes but just one.
260 // OTOH, because of the exponential decay, even when we need one slave, excess workload is always
261 // something like 0.95, in which case we want to allocate one node.
262 // so the threshold here is 1-MARGIN, and hence floor(excessWorkload+MARGIN) is needed to handle this.
263
264 Collection<PlannedNode> additionalCapacities = c.provision(label, (int)Math.round(Math.floor(excessWorkload+m)));
265 for (PlannedNode ac : additionalCapacities) {
266 excessWorkload -= ac.numExecutors;
267 LOGGER.info("Started provisioning "+ac.displayName+" from "+c.name+" with "+ac.numExecutors+" executors. Remaining excess workload:"+excessWorkload);
268 }
269 pendingLaunches.addAll(additionalCapacities);
270 }
271 }
272 }
273 }
274 }
275
276 /**
277 * Computes the threshold for triggering an allocation.
278 *
279 * <p>
280 * Because the excessive workload value is EMA, even when the snapshot value of the excessive
281 * workload is 1, the value never really gets to 1. So we need to introduce a notion of the margin M,
282 * where we provision a new node if the EMA of the excessive workload goes beyond 1-M (where M is a small value
283 * in the (0,1) range.)
284 *
285 * <p>
286 * M effectively controls how long Hudson waits until allocating a new node, in the face of workload.
287 * This delay is justified for absorbing temporary ups and downs, and can be interpreted as Hudson
288 * holding off provisioning in the hope that one of the existing nodes will become available.
289 *
290 * <p>
291 * M can be a constant value, but there's a benefit in adjusting M based on the total current capacity,
292 * based on the above justification; that is, if there's no existing capacity at all, holding off
293 * an allocation doesn't make much sense, as there won't be any executors available no matter how long we wait.
294 * On the other hand, if we have a large number of existing executors, chances are good that some
295 * of them become available &mdash; the chance gets better and better as the number of current total
296 * capacity increases.
297 *
298 * <p>
299 * Therefore, we compute the threshold margin as follows:
300 *
301 * <pre>
302 * M(t) = M* + (M0 - M*) alpha ^ t
303 * </pre>
304 *
305 * ... where:
306 *
307 * <ul>
308 * <li>M* is the ultimate margin value that M(t) converges to with t->inf,
309 * <li>M0 is the value of M(0), the initial value.
310 * <li>alpha is the decay factor in (0,1). M(t) converges to M* faster if alpha is smaller.
311 * </ul>
312 */
313 private float calcThresholdMargin(int totalSnapshot) {
314 float f = (float) (MARGIN + (MARGIN0 - MARGIN) * Math.pow(MARGIN_DECAY, totalSnapshot));
315 // defensively ensure that the threshold margin is in (0,1)
316 f = Math.max(f,0);
317 f = Math.min(f,1);
318 return f;
319 }
320
321 /**
322 * Periodically invoke NodeProvisioners
323 */
324 @Extension
325 public static class NodeProvisionerInvoker extends PeriodicWork {
326 /**
327 * Give some initial warm up time so that statically connected slaves
328 * can be brought online before we start allocating more.
329 */
330 public static int INITIALDELAY = Integer.getInteger(NodeProvisioner.class.getName()+".initialDelay",LoadStatistics.CLOCK*10);
331 public static int RECURRENCEPERIOD = Integer.getInteger(NodeProvisioner.class.getName()+".recurrencePeriod",LoadStatistics.CLOCK);
332
333 @Override
334 public long getInitialDelay() {
335 return INITIALDELAY;
336 }
337
338 public long getRecurrencePeriod() {
339 return RECURRENCEPERIOD;
340 }
341
342 @Override
343 protected void doRun() {
344 Jenkins h = Jenkins.getInstance();
345 h.unlabeledNodeProvisioner.update();
346 for( Label l : h.getLabels() )
347 l.nodeProvisioner.update();
348 }
349 }
350
351 private static final Logger LOGGER = Logger.getLogger(NodeProvisioner.class.getName());
352 private static final float MARGIN = Integer.getInteger(NodeProvisioner.class.getName()+".MARGIN",10)/100f;
353 private static final float MARGIN0 = Math.max(MARGIN, getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN0",0.5f));
354 private static final float MARGIN_DECAY = getFloatSystemProperty(NodeProvisioner.class.getName()+".MARGIN_DECAY",0.5f);
355
356 // TODO: picker should be selectable
357 private static final TimeScale TIME_SCALE = TimeScale.SEC10;
358
359 private static float getFloatSystemProperty(String propName, float defaultValue) {
360 String v = System.getProperty(propName);
361 if (v!=null)
362 try {
363 return Float.parseFloat(v);
364 } catch (NumberFormatException e) {
365 LOGGER.warning("Failed to parse a float value from system property "+propName+". value was "+v);
366 }
367 return defaultValue;
368 }
369 }
370