3
3
4
4
from __future__ import print_function
5
5
6
- import sys
7
- import os .path
8
6
import argparse
7
+ import copy
9
8
import errno
10
- import subprocess
11
9
import json
10
+ import os .path
12
11
import socket
12
+ import subprocess
13
+ import sys
13
14
import time
14
- import copy
15
15
16
16
PORT = 8642
17
17
18
18
CLIENT_CONFIG = {
19
- "options" : {"failByDrop" : False },
20
- "outdir" : "./reports/servers" ,
21
-
22
- "servers" : [
23
- {
24
- "agent" : "wsproto" ,
25
- "url" : "ws://localhost:{}" .format (PORT ),
26
- "options" : {"version" : 18 },
27
- },
28
- ],
29
-
30
- "cases" : ["*" ],
31
- "exclude-cases" : [],
32
- "exclude-agent-cases" : {},
19
+ "options" : {"failByDrop" : False },
20
+ "outdir" : "./reports/servers" ,
21
+ "servers" : [
22
+ {
23
+ "agent" : "wsproto" ,
24
+ "url" : "ws://localhost:{}" .format (PORT ),
25
+ "options" : {"version" : 18 },
26
+ }
27
+ ],
28
+ "cases" : ["*" ],
29
+ "exclude-cases" : [],
30
+ "exclude-agent-cases" : {},
33
31
}
34
32
35
33
SERVER_CONFIG = {
36
- "url" : "ws://localhost:{}" .format (PORT ),
37
-
38
- "options" : {"failByDrop" : False },
39
- "outdir" : "./reports/clients" ,
40
- "webport" : 8080 ,
41
-
42
- "cases" : ["*" ],
43
- "exclude-cases" : [],
44
- "exclude-agent-cases" : {}
34
+ "url" : "ws://localhost:{}" .format (PORT ),
35
+ "options" : {"failByDrop" : False },
36
+ "outdir" : "./reports/clients" ,
37
+ "webport" : 8080 ,
38
+ "cases" : ["*" ],
39
+ "exclude-cases" : [],
40
+ "exclude-agent-cases" : {},
45
41
}
46
42
47
43
CASES = {
48
44
"all" : ["*" ],
49
- "fast" :
45
+ "fast" : [
50
46
# The core functionality tests
51
- ["{}.*" .format (i ) for i in range (1 , 12 )]
47
+ * ["{}.*" .format (i ) for i in range (1 , 12 )],
52
48
# Compression tests -- in each section, the tests get progressively
53
49
# slower until they're taking 10s of seconds apiece. And it's
54
50
# mostly stress tests, without much extra coverage to show for
55
51
# it. (Weird trick: autobahntestsuite treats these as regexps
56
52
# except that . is quoted and * becomes .*)
57
- + ["12.*.[1234]$" , "13.*.[1234]$" ]
53
+ "12.*.[1234]$" ,
54
+ "13.*.[1234]$" ,
58
55
# At one point these were catching a unique bug that none of the
59
56
# above were -- they're relatively quick and involve
60
57
# fragmentation.
61
- + ["12.1.11" , "12.1.12" , "13.1.11" , "13.1.12" ],
58
+ "12.1.11" ,
59
+ "12.1.12" ,
60
+ "13.1.11" ,
61
+ "13.1.12" ,
62
+ ],
62
63
}
63
64
65
+
64
66
def say (* args ):
65
67
print ("run-autobahn-tests.py:" , * args )
66
68
69
+
67
70
def setup_venv ():
68
71
if not os .path .exists ("autobahntestsuite-venv" ):
69
72
say ("Creating Python 2.7 environment and installing autobahntestsuite" )
70
73
subprocess .check_call (
71
- ["virtualenv" , "-p" , "python2.7" , "autobahntestsuite-venv" ])
74
+ ["virtualenv" , "-p" , "python2.7" , "autobahntestsuite-venv" ]
75
+ )
72
76
subprocess .check_call (
73
- ["autobahntestsuite-venv/bin/pip" , "install" , "autobahntestsuite>=0.8.0" ])
77
+ ["autobahntestsuite-venv/bin/pip" , "install" , "autobahntestsuite>=0.8.0" ]
78
+ )
79
+
74
80
75
81
def wait_for_listener (port ):
76
82
while True :
@@ -87,13 +93,20 @@ def wait_for_listener(port):
87
93
finally :
88
94
sock .close ()
89
95
96
+
90
97
def coverage (command , coverage_settings ):
91
98
if not coverage_settings ["enabled" ]:
92
99
return [sys .executable ] + command
93
100
94
- return ([sys .executable , "-m" , "coverage" , "run" ,
95
- "--include" , coverage_settings ["wsproto-path" ]]
96
- + command )
101
+ return [
102
+ sys .executable ,
103
+ "-m" ,
104
+ "coverage" ,
105
+ "run" ,
106
+ "--include" ,
107
+ coverage_settings ["wsproto-path" ],
108
+ ] + command
109
+
97
110
98
111
def summarize (report_path ):
99
112
with open (os .path .join (report_path , "index.json" )) as f :
@@ -103,31 +116,36 @@ def summarize(report_path):
103
116
PASS = {"OK" , "INFORMATIONAL" }
104
117
for test_name , results in sorted (result_summary .items ()):
105
118
total += 1
106
- if (results ["behavior" ] not in PASS
107
- or results ["behaviorClose" ] not in PASS ):
119
+ if results ["behavior" ] not in PASS or results ["behaviorClose" ] not in PASS :
108
120
say ("FAIL:" , test_name , results )
109
121
say ("Details:" )
110
122
with open (os .path .join (report_path , results ["reportfile" ])) as f :
111
123
print (f .read ())
112
124
failed += 1
113
125
114
- speed_ordered = sorted (result_summary .items (),
115
- key = lambda kv : - kv [1 ]["duration" ])
126
+ speed_ordered = sorted (result_summary .items (), key = lambda kv : - kv [1 ]["duration" ])
116
127
say ("Slowest tests:" )
117
128
for test_name , results in speed_ordered [:5 ]:
118
129
say (" {}: {} seconds" .format (test_name , results ["duration" ] / 1000 ))
119
130
120
131
return failed , total
121
132
133
+
122
134
def run_client_tests (cases , coverage_settings ):
123
135
say ("Starting autobahntestsuite server" )
124
136
server_config = copy .deepcopy (SERVER_CONFIG )
125
137
server_config ["cases" ] = cases
126
138
with open ("auto-tests-server-config.json" , "w" ) as f :
127
139
json .dump (server_config , f )
128
140
server = subprocess .Popen (
129
- ["autobahntestsuite-venv/bin/wstest" , "-m" , "fuzzingserver" ,
130
- "-s" , "auto-tests-server-config.json" ])
141
+ [
142
+ "autobahntestsuite-venv/bin/wstest" ,
143
+ "-m" ,
144
+ "fuzzingserver" ,
145
+ "-s" ,
146
+ "auto-tests-server-config.json" ,
147
+ ]
148
+ )
131
149
say ("Waiting for server to start" )
132
150
wait_for_listener (PORT )
133
151
try :
@@ -143,6 +161,7 @@ def run_client_tests(cases, coverage_settings):
143
161
144
162
return summarize ("reports/clients" )
145
163
164
+
146
165
def run_server_tests (cases , coverage_settings ):
147
166
say ("Starting wsproto test server" )
148
167
server = subprocess .Popen (coverage (["./test_server.py" ], coverage_settings ))
@@ -156,8 +175,14 @@ def run_server_tests(cases, coverage_settings):
156
175
json .dump (client_config , f )
157
176
say ("Starting autobahntestsuite client" )
158
177
subprocess .check_call (
159
- ["autobahntestsuite-venv/bin/wstest" , "-m" , "fuzzingclient" ,
160
- "-s" , "auto-tests-client-config.json" ])
178
+ [
179
+ "autobahntestsuite-venv/bin/wstest" ,
180
+ "-m" ,
181
+ "fuzzingclient" ,
182
+ "-s" ,
183
+ "auto-tests-client-config.json" ,
184
+ ]
185
+ )
161
186
finally :
162
187
say ("Stopping server..." )
163
188
# Connection on this port triggers a shutdown
@@ -168,13 +193,12 @@ def run_server_tests(cases, coverage_settings):
168
193
169
194
return summarize ("reports/servers" )
170
195
196
+
171
197
def main ():
172
198
if not os .path .exists ("test_client.py" ):
173
199
say ("Run me from the compliance/ directory" )
174
200
sys .exit (2 )
175
- coverage_settings = {
176
- "coveragerc" : "../.coveragerc" ,
177
- }
201
+ coverage_settings = {"coveragerc" : "../.coveragerc" }
178
202
try :
179
203
import wsproto
180
204
except ImportError :
@@ -188,16 +212,16 @@ def main():
188
212
parser .add_argument ("MODE" , help = "'client' or 'server'" )
189
213
# can do e.g.
190
214
# --cases='["1.*"]'
191
- parser .add_argument ("--cases" ,
192
- help = "'fast' or 'all' or a JSON list" ,
193
- default = "fast" )
215
+ parser .add_argument (
216
+ "--cases" , help = "'fast' or 'all' or a JSON list" , default = "fast"
217
+ )
194
218
parser .add_argument ("--cov" , help = "enable coverage" , action = "store_true" )
195
219
196
220
args = parser .parse_args ()
197
221
198
222
coverage_settings ["enabled" ] = args .cov
199
223
cases = args .cases
200
- #pylint: disable=consider-using-get
224
+ # pylint: disable=consider-using-get
201
225
if cases in CASES :
202
226
cases = CASES [cases ]
203
227
else :
@@ -213,8 +237,9 @@ def main():
213
237
say ("Unrecognized mode, try 'client' or 'server'" )
214
238
sys .exit (2 )
215
239
216
- say ("in {} mode: failed {} out of {} total"
217
- .format (args .MODE .upper (), failed , total ))
240
+ say (
241
+ "in {} mode: failed {} out of {} total" .format (args .MODE .upper (), failed , total )
242
+ )
218
243
219
244
if failed :
220
245
say ("Test failed" )
0 commit comments