File D:\maven-qcr-plugin-testsunitaires\qcr4java\jenkins-master\target\site\qc\test\src\test\java\hudson\tasks\junit\TestResultPublishingTest.java.html

1 /*
2 * The MIT License
3 *
4 * Copyright (c) 2009, Yahoo!, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24 package hudson.tasks.junit;
25
26 import com.gargoylesoftware.htmlunit.Page;
27 import com.gargoylesoftware.htmlunit.html.*;
28 import com.gargoylesoftware.htmlunit.xml.XmlPage;
29 import hudson.FilePath;
30 import hudson.model.FreeStyleBuild;
31 import hudson.model.FreeStyleProject;
32 import hudson.model.Project;
33 import hudson.model.Run;
34 import hudson.slaves.DumbSlave;
35 import org.jvnet.hudson.test.Bug;
36 import org.jvnet.hudson.test.HudsonTestCase;
37 import org.jvnet.hudson.test.TouchBuilder;
38 import org.jvnet.hudson.test.recipes.LocalData;
39 import org.xml.sax.SAXException;
40
41 import java.io.IOException;
42 import java.util.List;
43 import java.util.concurrent.TimeUnit;
44
45 public class TestResultPublishingTest extends HudsonTestCase {
46 private FreeStyleProject project;
47 private JUnitResultArchiver archiver;
48 private final String BASIC_TEST_PROJECT = "percival";
49 private final String TEST_PROJECT_WITH_HISTORY = "wonky";
50
51
52 @Override
53 protected void setUp() throws Exception {
54 super.setUp();
55 project = createFreeStyleProject(BASIC_TEST_PROJECT);
56 archiver = new JUnitResultArchiver("*.xml");
57 project.getPublishersList().add(archiver);
58 project.getBuildersList().add(new TouchBuilder());
59 }
60
61 @LocalData
62 public void testBasic() throws Exception {
63 FreeStyleBuild build = project.scheduleBuild2(0).get(30, TimeUnit.SECONDS);
64
65 assertTestResults(build);
66
67 HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
68 wc.getPage(project); // project page
69 wc.getPage(build); // build page
70 wc.getPage(build, "testReport"); // test report
71 wc.getPage(build, "testReport/hudson.security"); // package
72 wc.getPage(build, "testReport/hudson.security/HudsonPrivateSecurityRealmTest/"); // class
73 wc.getPage(build, "testReport/hudson.security/HudsonPrivateSecurityRealmTest/testDataCompatibilityWith1_282/"); // method
74 }
75
76
77 @LocalData
78 public void testSlave() throws Exception {
79 DumbSlave s = createOnlineSlave();
80 project.setAssignedLabel(s.getSelfLabel());
81
82 FilePath src = new FilePath(jenkins.getRootPath(), "jobs/" + BASIC_TEST_PROJECT + "/workspace/");
83 assertNotNull(src);
84 FilePath dest = s.getWorkspaceFor(project);
85 assertNotNull(dest);
86 src.copyRecursiveTo("*.xml", dest);
87
88 testBasic();
89 }
90
91
92 /**
93 * Verify that we can successfully parse and display test results in the
94 * open junit test result publishing toolchain. Ensure that we meet this
95 * compatibility requirement:
96 * From users' point of view, Hudson core JUnit should continue
97 * to work as if nothing has changed
98 * - Old testReport URLs should still work
99 */
100 @LocalData
101 public void testOpenJUnitPublishing() throws IOException, SAXException {
102 List<Project> projects = this.jenkins.getProjects();
103 // Make sure there's a project named TEST_PROJECT_WITH_HISTORY
104 Project proj = null;
105 for (Project p : projects) {
106 if (p.getName().equals(TEST_PROJECT_WITH_HISTORY)) proj = p;
107 }
108 assertNotNull("We should have a project named " + TEST_PROJECT_WITH_HISTORY, proj);
109
110 // Validate that there are test results where I expect them to be:
111 HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
112
113 // On the project page:
114 HtmlPage projectPage = wc.getPage(proj);
115 // we should have a link that reads "Latest Test Result"
116 // that link should go to http://localhost:8080/job/breakable/lastBuild/testReport/
117 assertXPath(projectPage, "//a[@href='lastCompletedBuild/testReport/']");
118 assertXPathValue(projectPage, "//a[@href='lastCompletedBuild/testReport/']", "Latest Test Result");
119 assertXPathValueContains(projectPage, "//a[@href='lastCompletedBuild/testReport/']", "Latest Test Result");
120 // after "Latest Test Result" it should say "no failures"
121 assertXPathResultsContainText(projectPage, "//td", "(no failures)");
122 // there should be a test result trend graph
123 assertXPath(projectPage, "//img[@src='test/trend']");
124 // the trend graph should be served up with a good http status
125 Page trendGraphPage = wc.goTo(proj.getUrl() + "/test/trend", "image/png");
126 assertGoodStatus(trendGraphPage);
127
128 // The trend graph should be clickable and take us to a run details page
129 Object imageNode = projectPage.getFirstByXPath("//img[@src='test/trend']");
130 assertNotNull("couldn't find any matching nodes", imageNode);
131 assertTrue("image node should be an HtmlImage object", imageNode instanceof HtmlImage);
132 // TODO: Check that we can click on the graph and get to a particular run. How do I do this with HtmlUnit?
133
134 XmlPage xmlProjectPage = wc.goToXml(proj.getUrl() + "/lastBuild/testReport/api/xml");
135 assertXPath(xmlProjectPage, "/testResult");
136 assertXPath(xmlProjectPage, "/testResult/suite");
137 assertXPath(xmlProjectPage, "/testResult/failCount");
138 assertXPathValue(xmlProjectPage, "/testResult/failCount", "0");
139 assertXPathValue(xmlProjectPage, "/testResult/passCount", "4");
140 assertXPathValue(xmlProjectPage, "/testResult/skipCount", "0");
141 String[] packages = {"org.jvnet.hudson.examples.small.AppTest", "org.jvnet.hudson.examples.small.MiscTest", "org.jvnet.hudson.examples.small.deep.DeepTest"};
142 for (String packageName : packages) {
143 assertXPath(xmlProjectPage, "/testResult/suite/case/className[text()='" + packageName + "']");
144 }
145
146 // Go to a page that we know has a failure
147 HtmlPage buildPage = wc.getPage(proj.getBuildByNumber(3));
148 assertGoodStatus(buildPage);
149 // We expect to see one failure, for com.yahoo.breakable.misc.UglyTest.becomeUglier
150 // which should link to http://localhost:8080/job/wonky/3/testReport/org.jvnet.hudson.examples.small/MiscTest/testEleanor/
151 assertXPathResultsContainText(buildPage, "//a", "org.jvnet.hudson.examples.small.MiscTest.testEleanor");
152 HtmlAnchor failingTestLink = buildPage.getFirstAnchorByText("org.jvnet.hudson.examples.small.MiscTest.testEleanor");
153 assertNotNull(failingTestLink);
154 Page failingTestPage = failingTestLink.click();
155 assertGoodStatus(failingTestPage);
156
157 // Go to the xml page for a build we know has failures
158 XmlPage xmlBuildPage = wc.goToXml(proj.getBuildByNumber(3).getUrl() + "/api/xml");
159 assertXPathValue(xmlBuildPage, "//failCount", "2");
160 assertXPathValue(xmlBuildPage, "//skipCount", "0");
161 assertXPathValue(xmlBuildPage, "//totalCount", "4");
162 assertXPathValue(xmlBuildPage, "//result", "FAILURE");
163
164 // Check overall test result counts
165 XmlPage xmlTestReportPage = wc.goToXml(proj.getBuildByNumber(3).getUrl() + "/testReport/api/xml");
166 assertXPathValue(xmlTestReportPage, "/testResult/failCount", "2");
167 assertXPathValue(xmlTestReportPage, "/testResult/passCount", "2");
168 assertXPathValue(xmlTestReportPage, "/testResult/skipCount", "0");
169
170 // Make sure the right tests passed and failed
171 assertXPathValue(xmlTestReportPage, "/testResult/suite/case[className/text()='org.jvnet.hudson.examples.small.AppTest']/status", "PASSED");
172 assertXPathValue(xmlTestReportPage, "/testResult/suite/case[name/text()='testEleanor']/status", "FAILED");
173
174
175 // TODO: implement more of these tests
176 // On the lastBuild/testReport page:
177 // Breadcrumbs should read #6 > Test Result where Test Result is a link to this page
178 // inside of div id="main-panel" we should find the text "0 failures (-1)"
179 // we should have a blue bar which is blue all the way across: div style="width: 100%; height: 1em; background-color: rgb(114, 159, 207);
180 // we should find the words "7 tests (?0)"
181 // we should find the words "All Tests"
182 // we should find a table
183
184 // Inside that table, there should be the following rows:
185 // org.jvnet.hudson.examples.small 0ms 0 -1 0 3
186 // org.jvnet.hudson.examples.small.deep 4ms 0 0 0 1
187 Run theRun = proj.getBuildByNumber(7);
188 assertTestResultsAsExpected(wc, theRun, "/testReport",
189 "org.jvnet.hudson.examples.small", "0 ms", "SUCCESS",
190 /* total tests expected, diff */ 3, 0,
191 /* fail count expected, diff */ 0, -1,
192 /* skip count expected, diff */ 0, 0);
193
194 assertTestResultsAsExpected(wc, theRun, "/testReport",
195 "org.jvnet.hudson.examples.small.deep", "4 ms", "SUCCESS",
196 /* total tests expected, diff */ 1, 0,
197 /* fail count expected, diff */ 0, 0,
198 /* skip count expected, diff */ 0, 0);
199
200 // TODO: more, more, more.
201 // TODO: test report history by package
202
203 }
204
205 /**
206 * Test to demonstrate bug HUDSON-5246, inter-build diffs for junit test results are wrong
207 */
208 @Bug(5246)
209 @LocalData
210 public void testInterBuildDiffs() throws IOException, SAXException {
211 List<Project> projects = this.jenkins.getProjects();
212 // Make sure there's a project named TEST_PROJECT_WITH_HISTORY
213 Project proj = null;
214 for (Project p : projects) {
215 if (p.getName().equals(TEST_PROJECT_WITH_HISTORY)) proj = p;
216 }
217 assertNotNull("We should have a project named " + TEST_PROJECT_WITH_HISTORY, proj);
218
219 // Validate that there are test results where I expect them to be:
220 HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
221 Run theRun = proj.getBuildByNumber(4);
222 assertTestResultsAsExpected(wc, theRun, "/testReport",
223 "org.jvnet.hudson.examples.small", "12 ms", "FAILURE",
224 /* total tests expected, diff */ 3, 0,
225 /* fail count expected, diff */ 1, 0,
226 /* skip count expected, diff */ 0, 0);
227
228
229 }
230
231 /**
232 * Make sure the open junit publisher shows junit history
233 * @throws IOException
234 * @throws SAXException
235 */
236 @LocalData
237 public void testHistoryPageOpenJunit() throws IOException, SAXException {
238 List<Project> projects = this.jenkins.getProjects();
239 // Make sure there's a project named breakable
240 Project proj = null;
241 for (Project p : projects) {
242 if (p.getName().equals(TEST_PROJECT_WITH_HISTORY)) {
243 proj = p;
244 break;
245 }
246 }
247 assertNotNull("We should have a project named " + TEST_PROJECT_WITH_HISTORY, proj);
248
249 // Validate that there are test results where I expect them to be:
250 HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
251
252 HtmlPage historyPage = wc.getPage(proj.getBuildByNumber(7),"/testReport/history/");
253 assertGoodStatus(historyPage);
254 assertXPath(historyPage, "//img[@id='graph']");
255 assertXPath(historyPage, "//table[@id='testresult']");
256 HtmlElement wholeTable = historyPage.getElementById("testresult");
257 assertNotNull("table with id 'testresult' exists", wholeTable);
258 assertTrue("wholeTable is a table", wholeTable instanceof HtmlTable);
259 HtmlTable table = (HtmlTable) wholeTable;
260
261 // We really want to call table.getRowCount(), but
262 // it returns 1, not the real answer,
263 // because this table has *two* tbody elements,
264 // and getRowCount() only seems to count the *first* tbody.
265 // Maybe HtmlUnit can't handle the two tbody's. In any case,
266 // the tableText.contains tests do a (ahem) passable job
267 // of detecting whether the history results are present.
268
269 String tableText = table.getTextContent();
270 assertTrue("Table text is missing the project name",
271 tableText.contains(TEST_PROJECT_WITH_HISTORY));
272 assertTrue("Table text is missing the build number",
273 tableText.contains("7"));
274 assertTrue("Table text is missing the test duration",
275 tableText.contains("4 ms"));
276 }
277
278
279
280 void assertStringEmptyOrNull(String msg, String str) {
281 if (str==null)
282 return;
283 if (str.equals(""))
284 return;
285 fail(msg + "(should be empty or null) : \'" + str + "\'");
286 }
287
288 void assertPaneDiffText(String msg, int expectedValue, Object paneObj) {
289 assertTrue( "paneObj should be an HtmlElement", paneObj instanceof HtmlElement );
290 String paneText = ((HtmlElement) paneObj).asText();
291 if (expectedValue==0) {
292 assertStringEmptyOrNull(msg, paneText);
293 } else {
294 String expectedString =
295 (expectedValue >= 1 ? "+" : "-")
296 + Math.abs(expectedValue);
297 assertEquals(msg, expectedString, paneText);
298 }
299 }
300
301 void assertTestResultsAsExpected(WebClient wc, Run run, String restOfUrl,
302 String packageName,
303 String expectedResult, String expectedDurationStr,
304 int expectedTotalTests, int expectedTotalDiff,
305 int expectedFailCount, int expectedFailDiff,
306 int expectedSkipCount, int expectedSkipDiff) throws IOException, SAXException {
307
308 // TODO: verify expectedResult
309 // TODO: verify expectedDuration
310
311 XmlPage xmlPage = wc.goToXml(run.getUrl() + restOfUrl + "/" + packageName + "/api/xml");
312 int expectedPassCount = expectedTotalTests - expectedFailCount - expectedSkipCount;
313 // Verify xml results
314 assertXPathValue(xmlPage, "/packageResult/failCount", Integer.toString(expectedFailCount));
315 assertXPathValue(xmlPage, "/packageResult/skipCount", Integer.toString(expectedSkipCount));
316 assertXPathValue(xmlPage, "/packageResult/passCount", Integer.toString(expectedPassCount));
317 assertXPathValue(xmlPage, "/packageResult/name", packageName);
318
319 // TODO: verify html results
320 HtmlPage testResultPage = wc.getPage(run, restOfUrl);
321
322 // Verify inter-build diffs in html table
323 String xpathToFailDiff = "//table[@id='testresult']//tr[td//span[text()=\"" + packageName + "\"]]/td[4]";
324 String xpathToSkipDiff = "//table[@id='testresult']//tr[td//span[text()=\"" + packageName + "\"]]/td[6]";
325 String xpathToTotalDiff = "//table[@id='testresult']//tr[td//span[text()=\"" + packageName + "\"]]/td[last()]";
326
327 Object totalDiffObj = testResultPage.getFirstByXPath(xpathToTotalDiff);
328 assertPaneDiffText("total diff", expectedTotalDiff, totalDiffObj);
329
330 Object failDiffObj = testResultPage.getFirstByXPath(xpathToFailDiff);
331 assertPaneDiffText("failure diff", expectedFailDiff, failDiffObj);
332
333 Object skipDiffObj = testResultPage.getFirstByXPath(xpathToSkipDiff);
334 assertPaneDiffText("skip diff", expectedSkipDiff, skipDiffObj);
335
336 // TODO: The link in the table for each of the three packages in the testReport table should link to a by-package page,
337 // TODO: for example, http://localhost:8080/job/breakable/lastBuild/testReport/com.yahoo.breakable.misc/
338
339 }
340
341 // TODO: Make sure that we meet this compatibility requirement:
342 // TODO: From users' point of view, Open Source *Unit publishers should continue to work as if nothing has changed
343 // TODO: * Old testReport URLs should still work
344
345
346 private void assertTestResults(FreeStyleBuild build) {
347 TestResultAction testResultAction = build.getAction(TestResultAction.class);
348 assertNotNull("no TestResultAction", testResultAction);
349
350 TestResult result = testResultAction.getResult();
351 assertNotNull("no TestResult", result);
352
353 assertEquals("should have 1 failing test", 1, testResultAction.getFailCount());
354 assertEquals("should have 1 failing test", 1, result.getFailCount());
355
356 assertEquals("should have 132 total tests", 132, testResultAction.getTotalCount());
357 assertEquals("should have 132 total tests", 132, result.getTotalCount());
358 }
359
360 }
361