travis: always generate a list file; show subtest number

Always generate a list file and point output at a given filename, even
if failure is expected.

Remove ad hoc -o output names that generally don't point into safe
locations.

The result is that the preprocessing options (-E) no longer output to
stdout, so change the tests accordingly.

Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

.gitignore

@@ -14,6 +14,7 @@
 *.ith
 *.lib
 *.lst
+*.log
 *.map
 *.mo32
 *.mo64
@@ -122,3 +123,4 @@ TAGS
 /autoconf/config.*
 /autoconf/install-sh
 /autoconf/clean.sh
+/travis/test/_version.stdout

travis/nasm-t.py

@@ -147,6 +147,7 @@ def expand_templates(desc_array):
             for k, v in own.items():
                 desc_array[i][k] = v
             del desc_array[i]['id']
+        desc_array[i]['_seq'] = '.%d' % (i+1)
     return desc_array
 
 def prepare_desc(desc, basedir, name, path):
@@ -157,7 +158,10 @@ def prepare_desc(desc, basedir, name, path):
     desc['_base-dir'] = basedir
     desc['_json-file'] = name
     desc['_json-path'] = path
-    desc['_test-name'] = basedir + os.sep + name[:-5]
+    if not '_seq' in desc:
+        desc['_seq'] = ''
+    desc['_test-name'] = basedir + os.sep + name[:-5] + desc['_seq']
+
     #
     # If no target provided never update
     if 'target' not in desc:
@@ -338,19 +342,16 @@ def prepare_run_opts(desc):
         opts += ['-f', desc['format']]
     if 'option' in desc:
         opts += desc['option'].split(" ")
+    outfile = desc['_test-name'] + '.out'
     for t in desc['target']:
         if 'output' in t:
-            if 'option' in t:
-                opts += t['option'].split(" ")
-            else:
-                opts += ['-o']
             outfile = desc['_base-dir'] + os.sep + t['output']
-            opts += [outfile, '-L+', '-l', outfile + '.lst']
-        if 'stdout' in t or 'stderr' in t:
-            if 'option' in t:
-                opts += t['option'].split(" ")
+        if 'option' in t:
+            opts += t['option'].split(" ")
+    opts += ['-o', outfile, '-L+', '-l', outfile + '.lst']
     if 'source' in desc:
         opts += [desc['_base-dir'] + os.sep + desc['source']]
+
     return opts
 
 def exec_nasm(desc):
@@ -436,7 +437,7 @@ def test_run(desc):
             if f:
                 out_data = f_pat.sub(f_sub, out_data, 0)
             if cmp_std(match, match_data, 'stdout', out_data) == False:
-                return test_fail(desc['_test-name'], "Stdout mismatch")
+                return test_fail(desc['_test-name'], "stdout mismatch")
             else:
                 stdout = ""
         elif 'stderr' in t:
@@ -449,17 +450,17 @@ def test_run(desc):
             if f:
                 out_data = f_pat.sub(f_sub, out_data, 0)
             if cmp_std(match, match_data, 'stderr', out_data) == False:
-                return test_fail(desc['_test-name'], "Stderr mismatch")
+                return test_fail(desc['_test-name'], "stderr mismatch")
             else:
                 stderr = ""
 
     if stdout != "":
         show_std("stdout", stdout)
-        return test_fail(desc['_test-name'], "Stdout is not empty")
+        return test_fail(desc['_test-name'], "stdout is not empty")
 
     if stderr != "":
         show_std("stderr", stderr)
-        return test_fail(desc['_test-name'], "Stderr is not empty")
+        return test_fail(desc['_test-name'], "stderr is not empty")
 
     return test_pass(desc['_test-name'])
 

travis/test/align13.json

@@ -4,7 +4,7 @@
 		"id": "align13",
 		"format": "bin",
 		"source": "align13.asm",
-		"option": "-Ox -o align13.bin",
+		"option": "-Ox",
 		"target": [
 			{ "stderr": "align13.stderr" }
 		],
@@ -13,13 +13,13 @@
 	{
 		"description": "Test of non-power-of-2 alignment (-O1)",
 		"ref": "align13",
-		"option": "-O1 -o align13.bin",
+		"option": "-O1",
 		"update": "false"
 	},
 	{
 		"description": "Test of non-power-of-2 alignment (-O0)",
 		"ref": "align13",
-		"option": "-O0 -o align13.bin",
+		"option": "-O0",
 		"update": "false"
 	}
 ]

travis/test/align13s.json

@@ -4,7 +4,7 @@
 		"id": "align13s",
 		"format": "bin",
 		"source": "align13s.asm",
-		"option": "-Ox -o align13s.bin",
+		"option": "-Ox",
 		"target": [
 			{ "stderr": "align13s.stderr" }
 		],
@@ -13,13 +13,13 @@
 	{
 		"description": "Test of non-power-of-2 smart-alignment (-O1)",
 		"ref": "align13s",
-		"option": "-O1 -o align13s.bin",
+		"option": "-O1",
 		"update": "false"
 	},
 	{
 		"description": "Test of non-power-of-2 smart-alignment (-O0)",
 		"ref": "align13s",
-		"option": "-O0 -o align13s.bin",
+		"option": "-O0",
 		"update": "false"
 	}
 ]

travis/test/br2222615.json

@@ -12,7 +12,7 @@
 	{
 		"description": "Test br2222615 (err)",
 		"ref": "br2222615",
-		"option": "-DERROR -o br2222615.bin",
+		"option": "-DERROR",
 		"target": [
 			{ "stderr": "br2222615.stderr" }
 		],

travis/test/br3392392.json

@@ -12,7 +12,7 @@
 	{
 		"description": "Test br3392392 (error)",
 		"ref": "br3392392",
-		"option": "-DERROR -o br3392392.bin",
+		"option": "-DERROR",
 		"target": [
 			{ "stderr": "br3392392.stderr" }
 		],

travis/test/br3392528.json

@@ -4,7 +4,7 @@
 		"id": "br3392528",
 		"format": "bin",
 		"source": "br3392528.asm",
-		"option": "-Ox -o br3392528.bin",
+		"option": "-Ox",
 		"target": [
 			{ "stderr": "br3392528.stderr" }
 		],

travis/test/br3392707.json

@@ -13,7 +13,7 @@
 	{
 		"description": "Test br3392707 (error)",
 		"ref": "br3392707",
-		"option": "-o br3392707.bin -DERROR -Ox",
+		"option": "-DERROR -Ox",
 		"target": [
 			{ "stderr": "br3392707-error.stderr" }
 		],

travis/test/expimp.json

@@ -22,7 +22,7 @@
 	{
 		"description": "Test of explicitly and implicitly sized operands (-Ox)",
 		"ref": "expimp",
-		"option": "-Ox -DERROR -o expimp-ox-error.bin",
+		"option": "-Ox -DERROR",
 		"target": [
 			{ "stderr": "expimp-ox-error.stderr" }
 		],
@@ -31,7 +31,7 @@
 	{
 		"description": "Test of explicitly and implicitly sized operands (-O0 error)",
 		"ref": "expimp",
-		"option": "-O0 -DERROR -o expimp-o0-error.bin",
+		"option": "-O0 -DERROR",
 		"target": [
 			{ "stderr": "expimp-o0-error.stderr" }
 		],

travis/test/fwdoptpp.json

@@ -21,7 +21,7 @@
 	{
 		"description": "Test error directive",
 		"ref": "fwdoptpp",
-		"option": "-O0 -DERROR -o fwdoptpp.null.bin",
+		"option": "-O0 -DERROR",
 		"target": [
 			{ "stderr": "fwdoptpp.error.stderr" }
 		],
@@ -30,7 +30,7 @@
 	{
 		"description": "Test fatal directive",
 		"ref": "fwdoptpp",
-		"option": "-O0 -DFATAL -o fwdoptpp.null.bin",
+		"option": "-O0 -DFATAL",
 		"target": [
 			{ "stderr": "fwdoptpp.fatal.stderr" }
 		],

travis/test/macro-defaults.json

@@ -4,7 +4,7 @@
 		"id": "macro-defaults",
 		"format": "bin",
 		"source": "macro-defaults.asm",
-		"option": "-w+macro-defaults -o macro-defaults.bin",
+		"option": "-w+macro-defaults",
 		"target": [
 			{ "stderr": "macro-defaults-warn.stderr" }
 		]
@@ -12,7 +12,7 @@
 	{
 		"description": "Test warnings on macro default parameters (nowarn)",
 		"ref": "macro-defaults",
-		"option": "-w-macro-defaults -o macro-defaults.bin",
+		"option": "-w-macro-defaults",
 		"target": [
 			{ "stderr": "macro-defaults-nowarn.stderr" }
 		]

travis/test/mout.json

@@ -4,7 +4,7 @@
 		"id": "mout",
 		"format": "bin",
 		"source": "mout.asm",
-		"option": "-o mout.bin -o mout.bin",
+		"option": "-o mout.bin",
 		"target": [
 			{ "stderr": "mout.stderr" }
 		],

travis/test/mout.stderr

@@ -1,2 +1,2 @@
-mout.bin: fatal: more than one output file specified: mout.bin
+mout.bin: fatal: more than one output file specified: ./travis/test/mout.1.out
 

travis/test/nasmformat.json

@@ -6,7 +6,7 @@
 		"source": "nasmformat.asm",
 		"option": "-E -Ox",
 		"target": [
-			{ "stdout": "nasmformat-bin.stdout" }
+			{ "output": "nasmformat-bin.out" }
 		]
 	},
 	{
@@ -14,7 +14,7 @@
 		"ref": "nasmformat",
 		"format": "elf32",
 		"target": [
-			{ "stdout": "nasmformat-elf32.stdout" }
+			{ "output": "nasmformat-elf32.out" }
 		]
 	},
 	{
@@ -22,7 +22,7 @@
 		"ref": "nasmformat",
 		"format": "elf64",
 		"target": [
-			{ "stdout": "nasmformat-elf64.stdout" }
+			{ "output": "nasmformat-elf64.out" }
 		]
 	},
 	{
@@ -30,7 +30,7 @@
 		"ref": "nasmformat",
 		"format": "macho32",
 		"target": [
-			{ "stdout": "nasmformat-macho32.stdout" }
+			{ "output": "nasmformat-macho32.out" }
 		]
 	},
 	{
@@ -38,7 +38,7 @@
 		"ref": "nasmformat",
 		"format": "macho64",
 		"target": [
-			{ "stdout": "nasmformat-macho64.stdout" }
+			{ "output": "nasmformat-macho64.out" }
 		]
 	},
 	{
@@ -46,7 +46,7 @@
 		"ref": "nasmformat",
 		"format": "aout",
 		"target": [
-			{ "stdout": "nasmformat-aout.stdout" }
+			{ "output": "nasmformat-aout.out" }
 		]
 	},
 	{
@@ -54,7 +54,7 @@
 		"ref": "nasmformat",
 		"format": "win32",
 		"target": [
-			{ "stdout": "nasmformat-win32.stdout" }
+			{ "output": "nasmformat-win32.out" }
 		]
 	},
 	{
@@ -62,7 +62,7 @@
 		"ref": "nasmformat",
 		"format": "win64",
 		"target": [
-			{ "stdout": "nasmformat-win64.stdout" }
+			{ "output": "nasmformat-win64.out" }
 		]
 	},
 	{
@@ -70,7 +70,7 @@
 		"ref": "nasmformat",
 		"format": "obj",
 		"target": [
-			{ "stdout": "nasmformat-obj.stdout" }
+			{ "output": "nasmformat-obj.out" }
 		]
 	},
 	{
@@ -78,7 +78,7 @@
 		"ref": "nasmformat",
 		"format": "ieee",
 		"target": [
-			{ "stdout": "nasmformat-ieee.stdout" }
+			{ "output": "nasmformat-ieee.out" }
 		]
 	}
 ]

travis/test/ppindirect.json

@@ -6,7 +6,7 @@
 		"source": "ppindirect.asm",
 		"option": "-E -Ox",
 		"target": [
-			{ "stdout": "ppindirect.stdout" }
+			{ "output": "ppindirect.out" }
 		]
 	}
 ]

travis/test/pragma.json

@@ -6,7 +6,7 @@
 		"source": "pragma.asm",
 		"option": "-Ox -E",
 		"target": [
-			{ "stdout": "pragma.stdout" }
+			{ "output": "pragma.out" }
 		]
 	}
 ]

travis/test/ret.json

@@ -12,7 +12,7 @@
 	{
 		"description": "Test all the flavors of RET (err 16 bit)",
 		"ref": "ret",
-		"option": "-DERROR -DTEST_BITS_16 -o ret.bin",
+		"option": "-DERROR -DTEST_BITS_16",
 		"target": [
 			{ "stderr": "ret-16.stderr" }
 		],
@@ -21,7 +21,7 @@
 	{
 		"description": "Test all the flavors of RET (err 32 bit)",
 		"ref": "ret",
-		"option": "-DERROR -DTEST_BITS_32 -o ret.bin",
+		"option": "-DERROR -DTEST_BITS_32",
 		"target": [
 			{ "stderr": "ret-32.stderr" }
 		],
@@ -30,7 +30,7 @@
 	{
 		"description": "Test all the flavors of RET (err 64 bit)",
 		"ref": "ret",
-		"option": "-DERROR -DTEST_BITS_64 -o ret.bin",
+		"option": "-DERROR -DTEST_BITS_64",
 		"target": [
 			{ "stderr": "ret-64.stderr" }
 		],

travis/test/v4.json

@@ -12,7 +12,7 @@
 	{
 		"description": "Test v4 instructions (error)",
 		"ref": "v4",
-		"option": "-DERROR -o v4.bin",
+		"option": "-DERROR",
 		"target": [
 			{ "stderr": "v4.stderr" }
 		],

travis/test/vex.json

@@ -12,7 +12,7 @@
 	{
 		"description": "Test early REX/VEX prefix errors",
 		"ref": "vex",
-		"option": "-Ox -DERROR=1 -o vex1.bin.err",
+		"option": "-Ox -DERROR=1",
 		"target": [
 			{ "stderr": "vex1.stderr" }
 		],
@@ -21,7 +21,7 @@
 	{
 		"description": "Test late REX/VEX prefix errors",
 		"ref": "vex",
-		"option": "-Ox -DERROR=2 -o vex2.bin.err",
+		"option": "-Ox -DERROR=2",
 		"target": [
 			{ "stderr": "vex2.stderr" }
 		],