Improving test infrastructure

SRWebSocketTests/SRTAutobahnTests.m

@@ -15,6 +15,7 @@
 //
 
 #import <SenTestingKit/SenTestingKit.h>
+#import <SenTestingKit/SenTestRun.h>
 #import "SRWebSocket.h"
 #import "SRTWebSocketOperation.h"
 #import "SenTestCase+SRTAdditions.h"
@@ -37,6 +38,20 @@
 
 @end
 
+@interface NSInvocation (SRTBlockInvocation)
+
++ (NSInvocation *)invocationWithBlock:(dispatch_block_t)block;
+
+@end
+
+@interface SRTBlockInvoker
+
+- (id)initWithBlock:(dispatch_block_t)block;
+
+- (void)invoke;
+
+@end
+
 @interface UpdateOperation : SRTWebSocketOperation <SRWebSocketDelegate>
 
 - (id)initWithBaseURL:(NSURL *)url agent:(NSString *)agent;
@@ -48,16 +63,30 @@
     NSInteger _testCount;
     NSInteger _curTest;
     NSMutableArray *_sockets;
+    NSString *_testURLString;
+    NSURL *_prefixURL;
+    NSString *_agent;
+}
+
+- (id)initWithInvocation:(NSInvocation *)anInvocation;
+{
+    self = [super initWithInvocation:anInvocation];
+    if (self) {
+        _testURLString = [[NSProcessInfo processInfo].environment objectForKey:@"SR_TEST_URL"];
+        _prefixURL = [NSURL URLWithString:_testURLString];
+        _agent = [NSBundle bundleForClass:[self class]].bundleIdentifier;
+    }
+    return self;
 }
 
-- (void)testFuzzer;
+- (unsigned int)testCaseCount;
 {
-    _sockets = [[NSMutableArray alloc] init];
+    if (self.invocation) {
+        return [super testCaseCount];
+    }
     
-    NSString *testURLString = [[NSProcessInfo processInfo].environment objectForKey:@"SR_TEST_URL"];
-    NSURL *prefixURL = [NSURL URLWithString:testURLString];    
+    CaseGetterOperation *caseGetter = [[CaseGetterOperation alloc] initWithBaseURL:_prefixURL];
     
-    CaseGetterOperation *caseGetter = [[CaseGetterOperation alloc] initWithBaseURL:prefixURL];
     [caseGetter start];
     
     [self runCurrentRunLoopUntilTestPasses:^BOOL{
@@ -68,29 +97,87 @@
     
     NSInteger caseCount = caseGetter.caseCount;
     
+    return caseCount;
+}
+
+- (BOOL) isEmpty;
+{
+    return NO;
+}
+
+- (void) performTest:(SenTestRun *) aRun
+{
+    if (self.invocation) {
+        [super performTest:aRun];
+        return;
+    }
+    for (int i = 0; i < aRun.test.testCaseCount; i++) {
+        SEL sel = @selector(performTestWithNumber:);
+        NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:[[self class] instanceMethodSignatureForSelector:sel]];
+        
+        invocation.selector = sel;
+        invocation.target = self;
+        
+        [invocation setArgument:&i atIndex:2];
+        
+        SenTestCase *testCase = [[[self class] alloc] initWithInvocation:invocation];
+        
+        SenTestCaseRun *run = [[SenTestCaseRun alloc] initWithTest:testCase];
+        
+        [testCase performTest:run];
+    }
+    
+    [self updateReports];
+}
+
+- (NSInteger)testNum;
+{
+    NSInteger i;
+    [self.invocation getArgument:&i atIndex:2];
+    return i;
+}
+
+- (NSString *)description;
+{
+    if (self.invocation) {
+        return [NSString stringWithFormat:@"Autobahn test %d", self.testNum];
+    } else {
+        return @"Autobahn Test Harness";
+    }
+}
+
++ (id) defaultTestSuite
+{
+    return [[[self class] alloc] init];
+}
+
+- (void)performTestWithNumber:(NSInteger)testNumber;
+{
     NSOperationQueue *testQueue = [[NSOperationQueue alloc] init];
     
     testQueue.maxConcurrentOperationCount = 1;
     
-    NSString *agent = [NSBundle bundleForClass:[self class]].bundleIdentifier;
     
-    UpdateOperation *updateReportOperation = [[UpdateOperation alloc] initWithBaseURL:prefixURL agent:agent];
     
-    for (int caseNumber = 1; caseNumber <= caseCount; caseNumber++) {
-        TestOperation *testOp = [[TestOperation alloc] initWithBaseURL:prefixURL testNumber:caseNumber agent:agent];
-        [updateReportOperation addDependency:testOp];
-        [testQueue addOperation:testOp];
-    }
+    TestOperation *testOp = [[TestOperation alloc] initWithBaseURL:_prefixURL testNumber:testNumber agent:_agent];
+    [testQueue addOperation:testOp];
     
     testQueue.suspended = NO;
     
-    [testQueue addOperation:updateReportOperation];
+}
+
+- (void)updateReports;
+{
+    UpdateOperation *updateReportOperation = [[UpdateOperation alloc] initWithBaseURL:_prefixURL agent:_agent];
+    
+    [updateReportOperation start];
     
     [self runCurrentRunLoopUntilTestPasses:^BOOL{
         return updateReportOperation.isFinished;
     } timeout:60 * 60];
     
     STAssertNil(updateReportOperation.error, @"Updating the report should not have errored");
+
 }
 
 @end
@@ -112,12 +199,6 @@
     return self;
 }
 
-- (void)start;
-{
-    [super start];
-    NSLog(@"Starting test %d", _testNumber);
-}
-
 - (void)webSocket:(SRWebSocket *)webSocket didReceiveMessage:(id)message;
 {
     [webSocket send:message];
@@ -165,6 +246,3 @@
 @end
 
 
-
-
-

SRWebSocketTests/SRTWebSocketOperation.m

@@ -41,7 +41,6 @@
 
 - (void)start;
 {
-    NSLog(@"Starting %@", _url);
     dispatch_async(dispatch_get_main_queue(), ^{
         _webSocket = [[SRWebSocket alloc] initWithURLRequest:[NSURLRequest requestWithURL:_url]];
         _webSocket.delegate = self;

SRWebSocketTests/SenTestCase+SRTAdditions.h

@@ -20,7 +20,7 @@
 typedef BOOL (^PXPredicateBlock)();
 
 
-@interface SenTestCase (PXAdditions)
+@interface SenTest (PXAdditions)
 
 - (void)runCurrentRunLoopUntilTestPasses:(PXPredicateBlock)predicate timeout:(NSTimeInterval)timeout;
 

SocketRocket.xcodeproj/xcshareddata/xcschemes/SocketRocket.xcscheme

@@ -20,11 +20,25 @@
                ReferencedContainer = "container:SocketRocket.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "NO"
+            buildForArchiving = "NO"
+            buildForAnalyzing = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "F6BDA801145900D200FE3253"
+               BuildableName = "SRWebSocketTests.octest"
+               BlueprintName = "SRWebSocketTests"
+               ReferencedContainer = "container:SocketRocket.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
       </BuildActionEntries>
    </BuildAction>
    <TestAction
-      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.GDB"
-      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.GDB"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       shouldUseLaunchSchemeArgsEnv = "NO"
       buildConfiguration = "Debug">
       <PreActions>

SocketRocket.xcodeproj/xcshareddata/xcschemes/TestChat.xcscheme

@@ -41,7 +41,7 @@
    </TestAction>
    <LaunchAction
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
-      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.GDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       launchStyle = "0"
       useCustomWorkingDirectory = "NO"
       buildConfiguration = "Debug"

TestSupport/run_test.sh

@@ -16,9 +16,10 @@ sr-testharness -i '' -c "$TEST_SCENARIOS" &
 
 CHILD_PID=$!
 
-extra_opts="VALID_ARCHS=i386"
+extra_opts="VALID_ARCHS=i386 ARCH=i386"
 
-xcodebuild -target SocketRocket -arch i386 -configuration $CONFIGURATION -sdk iphonesimulator clean $extra_opts
-xcodebuild -target SRWebSocketTests -arch i386 -configuration $CONFIGURATION -sdk iphonesimulator clean build TEST_AFTER_BUILD=YES $extra_opts
+SHARED_ARGS="-arch i386 -configuration $CONFIGURATION -sdk iphonesimulator"
+
+xcodebuild -scheme SocketRocketTests $SHARED_ARGS TEST_AFTER_BUILD=YES  clean build $extra_opts
 
 kill $CHILD_PID

pages

@@ -1 +1 @@
-Subproject commit 8c217fe65cea5a5c7f52e100432aaf940488b2d8
+Subproject commit bc6c4de9546a54d0003c6598a8de07799b945243