From 44796e3bde6665e47cfd2ac23713e4b984e0016a Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 20 Mar 2018 14:15:47 +1300 Subject: [PATCH 01/24] gitlab-ci: Create swap space to work around the 2G image Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60accd463a7..287011cd568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,12 @@ # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options before_script: - - echo "Build starting ..." + - echo "Build starting (preparing swap)..." + - if [ $(df -m / --output=avail | tail -n1) -gt 10240 ]; then + sudo dd if=/dev/zero of=/samba-swap bs=1M count=6144; + sudo mkswap /samba-swap; + sudo swapon /samba-swap; + fi build_samba: stage: build From eb144575b9a01b12de20ea8d0cd53a88d762115e Mon Sep 17 00:00:00 2001 From: Joe Guo Date: Tue, 28 Nov 2017 10:20:54 +1300 Subject: [PATCH 02/24] gitlab-ci: set docker image and change tag 1. Specify docker image to use in gitlab-ci.yml 2. Change tag autobuild to docker So we can use gitlab.com shared runners. Signed-off-by: Joe Guo Reviewed-by: Andrew Bartlett --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 287011cd568..ef1cddad45b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,7 @@ # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options +image: registry.gitlab.com/samba-team/samba:latest + before_script: - echo "Build starting (preparing swap)..." - if [ $(df -m / --output=avail | tail -n1) -gt 10240 ]; then @@ -11,7 +13,7 @@ before_script: build_samba: stage: build tags: - - autobuild + - docker script: # this one takes about 4 hours to finish - python script/autobuild.py samba --verbose --tail --testbase /tmp/samba-testbase @@ -27,7 +29,7 @@ build_samba_none_env: build_samba_others: stage: build tags: - - autobuild + - docker script: - python script/autobuild.py samba-nopython --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py samba-systemkrb5 --verbose --tail --testbase /tmp/samba-testbase @@ -39,7 +41,7 @@ build_samba_others: build_ctdb: stage: build tags: - - autobuild + - docker script: - python script/autobuild.py samba-ctdb --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py ctdb --verbose --tail --testbase /tmp/samba-testbase @@ -47,7 +49,7 @@ build_ctdb: build_others: stage: build tags: - - autobuild + - docker script: - python script/autobuild.py ldb --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py pidl --verbose --tail --testbase /tmp/samba-testbase From 7d29c61d4dcf689d864c6df70c57388528fac604 Mon Sep 17 00:00:00 2001 From: Joe Guo Date: Tue, 12 Dec 2017 14:26:07 +1300 Subject: [PATCH 03/24] gitlab-ci: add git variable to speed up clone Set GIT_STATEGY to fetch, and GIT_DEPTH to 3. This will speed up cloning for repos. Signed-off-by: Joe Guo Reviewed-by: Andrew Bartlett --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef1cddad45b..c6739a5db34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,10 @@ image: registry.gitlab.com/samba-team/samba:latest +variables: + GIT_STRATEGY: fetch + GIT_DEPTH: "3" + before_script: - echo "Build starting (preparing swap)..." - if [ $(df -m / --output=avail | tail -n1) -gt 10240 ]; then From 89b4a98a9de1c4d4ef81678da38a0d5f95a9c77b Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 30 Jun 2017 11:15:40 +1200 Subject: [PATCH 04/24] autobuild: Run nt4_dc and nt4_member tests in parallel These do not interact with the main AD DC environments, so can run in parallel Signed-off-by: Andrew Bartlett --- script/autobuild.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index 699fb41df1f..7712d10ddfc 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -26,6 +26,7 @@ builddirs = { "ctdb" : "ctdb", "samba" : ".", + "samba-nt4" : ".", "samba-xc" : ".", "samba-o3" : ".", "samba-ctdb" : ".", @@ -48,6 +49,7 @@ defaulttasks = [ "ctdb", "samba", + "samba-nt4", "samba-xc", "samba-o3", "samba-ctdb", @@ -90,16 +92,25 @@ ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], - # We have 'test' before 'install' because, 'test' should work without 'install' + # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)' "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), ("test", "make test FAIL_IMMEDIATELY=1 " - "TESTS='--exclude-env=none'", - "text/plain"), + "TESTS='--exclude-env=none " + "--exclude-env=nt4_dc " + "--exclude-env=nt4_member'", "text/plain"), ("install", "make install", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], + # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long) + "samba-nt4" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"), + ("install", "make install", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), + ("clean", "make clean", "text/plain") ], + "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ], From abff7e03d6c0c6c88e3c0ec04ee46e32c5b2c003 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 30 Jun 2017 11:44:58 +1200 Subject: [PATCH 05/24] travis-ci: Run new samba-nt4 environment Signed-off-by: Andrew Bartlett --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 19d5b4ce0f8..7c092660707 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: - TASK=samba-none-env - TASK=samba-nopython - TASK=samba-systemkrb5 + - TASK=samba-nt4 - TASK=ldb - TASK=tdb - TASK=talloc From 69038fa74647351c231cdbab7a8cce00ca941272 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 5 Dec 2017 11:34:08 +1300 Subject: [PATCH 06/24] gitlab-ci: Add samba-nt4 environment to the CI This parallel build is de-coupled from the main samba build Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6739a5db34..9e3b11271a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,14 @@ build_samba: # this one takes about 4 hours to finish - python script/autobuild.py samba --verbose --tail --testbase /tmp/samba-testbase +build_samba_nt4: + stage: build + tags: + - autobuild + script: + # this one takes about 1 hours to finish + - python script/autobuild.py samba-nt4 --verbose --tail --testbase /tmp/samba-testbase + build_samba_none_env: stage: build tags: From 6cd6cd25d6a80cd6107a76bf9ca4603690852bba Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Sat, 3 Mar 2018 21:03:11 +1300 Subject: [PATCH 07/24] selftest: Do not run smb2.notify against nt4_dc and ad_dc This is a slow test and we need to keep the time on ad_dc down to below 50mins total for travis-ci. Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index c93abf5e632..a65edcea713 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -519,7 +519,6 @@ def plansmbtorture4testsuite(name, env, options, description=''): plansmbtorture4testsuite(t, "simpleserver", '//$SERVER/tmp -U$USERNAME%$PASSWORD') elif t == "smb2.notify": plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --signing=required') - plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD --signing=required') elif t == "smb2.dosmode": plansmbtorture4testsuite(t, "simpleserver", '//$SERVER/dosmode -U$USERNAME%$PASSWORD') elif t == "smb2.kernel-oplocks": From fe8b99dc8026929fc172fc87f67ad4dfc7f22148 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 30 Jun 2017 11:13:55 +1200 Subject: [PATCH 08/24] autobuild: Run all "ad_dc" environment tests in samba-ad-dc This allows us not to run ad_dc tests in the main build, making the autobuild process faster. The ad_dc tests run in less than 50mins on travis-ci, which allows this part of the tests to be run. Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 8 ++++++++ .travis.yml | 1 + script/autobuild.py | 12 +++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e3b11271a9..aac810cb992 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,14 @@ build_samba_nt4: # this one takes about 1 hours to finish - python script/autobuild.py samba-nt4 --verbose --tail --testbase /tmp/samba-testbase +build_samba_ad_dc: + stage: build + tags: + - autobuild + script: + # this one takes about 1 hours to finish + - python script/autobuild.py samba-ad-dc --verbose --tail --testbase /tmp/samba-testbase + build_samba_none_env: stage: build tags: diff --git a/.travis.yml b/.travis.yml index 7c092660707..acd6f484100 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ env: - TASK=samba-nopython - TASK=samba-systemkrb5 - TASK=samba-nt4 + - TASK=samba-ad-dc - TASK=ldb - TASK=tdb - TASK=talloc diff --git a/script/autobuild.py b/script/autobuild.py index 7712d10ddfc..0294afaae3e 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -34,6 +34,7 @@ "samba-static" : ".", "samba-test-only" : ".", "samba-none-env" : ".", + "samba-ad-dc" : ".", "samba-systemkrb5" : ".", "samba-nopython" : ".", "ldb" : "lib/ldb", @@ -56,6 +57,7 @@ "samba-libs", "samba-static", "samba-none-env", + "samba-ad-dc", "samba-systemkrb5", "samba-nopython", "ldb", @@ -98,7 +100,8 @@ ("test", "make test FAIL_IMMEDIATELY=1 " "TESTS='--exclude-env=none " "--exclude-env=nt4_dc " - "--exclude-env=nt4_member'", "text/plain"), + "--exclude-env=nt4_member " + "--exclude-env=ad_dc ", "text/plain"), ("install", "make install", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], @@ -111,6 +114,13 @@ ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], + # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) + "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], + "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ], From e7505ee63dded203c28a30d68cd864ecc5ae6e60 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Sun, 4 Mar 2018 21:32:09 +1300 Subject: [PATCH 09/24] autobuild: Remove fileserver tests from the main build Again, this is to allow these to run in the 50min timelimit of travis-ci and so gain test coverage. Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 8 ++++++++ .travis.yml | 1 + script/autobuild.py | 13 ++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac810cb992..7fc023e4bfc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,14 @@ build_samba_nt4: # this one takes about 1 hours to finish - python script/autobuild.py samba-nt4 --verbose --tail --testbase /tmp/samba-testbase +build_samba_fileserver: + stage: build + tags: + - autobuild + script: + # this one takes about 1 hours to finish + - python script/autobuild.py samba-fileserver --verbose --tail --testbase /tmp/samba-testbase + build_samba_ad_dc: stage: build tags: diff --git a/.travis.yml b/.travis.yml index acd6f484100..2ef0a0b1653 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ env: - TASK=samba-nopython - TASK=samba-systemkrb5 - TASK=samba-nt4 + - TASK=samba-fileserver - TASK=samba-ad-dc - TASK=ldb - TASK=tdb diff --git a/script/autobuild.py b/script/autobuild.py index 0294afaae3e..7b3d4bf6b35 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -27,6 +27,7 @@ "ctdb" : "ctdb", "samba" : ".", "samba-nt4" : ".", + "samba-fileserver" : ".", "samba-xc" : ".", "samba-o3" : ".", "samba-ctdb" : ".", @@ -51,6 +52,7 @@ defaulttasks = [ "ctdb", "samba", "samba-nt4", + "samba-fileserver", "samba-xc", "samba-o3", "samba-ctdb", @@ -101,7 +103,9 @@ "TESTS='--exclude-env=none " "--exclude-env=nt4_dc " "--exclude-env=nt4_member " - "--exclude-env=ad_dc ", "text/plain"), + "--exclude-env=ad_dc " + "--exclude-env=fileserver'", + "text/plain"), ("install", "make install", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], @@ -114,6 +118,13 @@ ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], + # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) + "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], + # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), From e4db2b26f5c3f5e016f615ac7890cd24837d1e52 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Sun, 4 Mar 2018 21:20:31 +1300 Subject: [PATCH 10/24] selftest: Move base.delaywrite tests to fileserver environment This aims to keep the ad_dc tests well below 50mins for travis CI and base.delaywrite is very slow. Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index a65edcea713..9ac8233aa6c 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -405,7 +405,7 @@ def plansmbtorture4testsuite(name, env, options, description=''): for t in tests: if t == "base.delaywrite": - plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD -k yes --maximum-runtime=900') + plansmbtorture4testsuite(t, "fileserver", '//$SERVER/tmp -U$USERNAME%$PASSWORD --maximum-runtime=900') elif t == "base.createx_access": plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD -k yes --maximum-runtime=900') elif t == "rap.sam": From aecef8de1a700533cfaefabcb8fce24ef732122e Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Sun, 4 Mar 2018 21:33:50 +1300 Subject: [PATCH 11/24] Move smbtorture3 tests to fileserver environment Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index 9ac8233aa6c..393615cb77c 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -85,7 +85,7 @@ def plansmbtorture4testsuite(name, env, options, description=''): "BAD-NBT-SESSION"] for t in tests: - plantestsuite("samba3.smbtorture_s3.plain(nt4_dc).%s" % t, "nt4_dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"]) + plantestsuite("samba3.smbtorture_s3.plain(fileserver).%s" % t, "fileserver", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"]) plantestsuite("samba3.smbtorture_s3.crypt_client(nt4_dc).%s" % t, "nt4_dc", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/tmp', '$USERNAME', '$PASSWORD', smbtorture3, "-e", "-l $LOCAL_PATH"]) if t == "TORTURE": # this is a negative test to verify that the server rejects From 6bc34aaa4eb643b2acf4d12c09cc906ace4ef1d0 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Sun, 4 Mar 2018 21:36:22 +1300 Subject: [PATCH 12/24] autobuild: Try and test different configure options for new environments Signed-off-by: Andrew Bartlett --- script/autobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/autobuild.py b/script/autobuild.py index 7b3d4bf6b35..6e8abe77479 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -120,7 +120,7 @@ # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("configure", "./configure.developer --without-ad-dc --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], From eff2110aa30aaccbea57d596d121f6bf5ce12a80 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Mon, 5 Mar 2018 08:04:22 +1300 Subject: [PATCH 13/24] selftest: Move slower base.deny1 and base.deny2 to fileserver environment This avoids these running in the ad_dc environment which we need to get under 50mins for travis-ci. Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index 393615cb77c..00e04318c7c 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -404,7 +404,7 @@ def plansmbtorture4testsuite(name, env, options, description=''): tests= base + raw + smb2 + rpc + unix + local + rap + nbt + libsmbclient + idmap + vfs for t in tests: - if t == "base.delaywrite": + if t == "base.delaywrite" or t == "base.deny1" or t == "base.deny2": plansmbtorture4testsuite(t, "fileserver", '//$SERVER/tmp -U$USERNAME%$PASSWORD --maximum-runtime=900') elif t == "base.createx_access": plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD -k yes --maximum-runtime=900') From 4e1dd0e58dc94b5149b4acfeb0a00e2f0f2da3d8 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Mon, 5 Mar 2018 17:08:51 +1300 Subject: [PATCH 14/24] selftest: Move samba.tests.samba_tool{.dnscmd,.sites} to chgdcpass This helps reduce the runtime of ad_dc which needs to be under 50mins including build time to run on travis-ci. Signed-off-by: Andrew Bartlett --- source4/selftest/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 700decd8b9d..adbca195c2a 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -612,8 +612,9 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check") planpythontestsuite("none", "samba.tests.samba_tool.help") -planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.sites") -planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.dnscmd") +# Run these against chgdcpass to share the runtime load +planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.sites") +planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.dnscmd") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.rpcecho") From a97ac6cdc8f83ba35978d0a3bb1ad73c53f16cc1 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 20 Mar 2018 15:07:03 +1300 Subject: [PATCH 15/24] gitlab-ci: Set shared and private tags to allow builds that need ext4 to pass Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7fc023e4bfc..6846b2a782d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ build_samba: stage: build tags: - docker + - private script: # this one takes about 4 hours to finish - python script/autobuild.py samba --verbose --tail --testbase /tmp/samba-testbase @@ -25,7 +26,8 @@ build_samba: build_samba_nt4: stage: build tags: - - autobuild + - docker + - private script: # this one takes about 1 hours to finish - python script/autobuild.py samba-nt4 --verbose --tail --testbase /tmp/samba-testbase @@ -33,7 +35,8 @@ build_samba_nt4: build_samba_fileserver: stage: build tags: - - autobuild + - docker + - shared script: # this one takes about 1 hours to finish - python script/autobuild.py samba-fileserver --verbose --tail --testbase /tmp/samba-testbase @@ -41,7 +44,8 @@ build_samba_fileserver: build_samba_ad_dc: stage: build tags: - - autobuild + - docker + - private script: # this one takes about 1 hours to finish - python script/autobuild.py samba-ad-dc --verbose --tail --testbase /tmp/samba-testbase @@ -49,7 +53,8 @@ build_samba_ad_dc: build_samba_none_env: stage: build tags: - - autobuild + - docker + - shared script: # this one takes about 1 hours to finish - python script/autobuild.py samba-none-env --verbose --tail --testbase /tmp/samba-testbase @@ -58,6 +63,7 @@ build_samba_others: stage: build tags: - docker + - shared script: - python script/autobuild.py samba-nopython --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py samba-systemkrb5 --verbose --tail --testbase /tmp/samba-testbase @@ -70,6 +76,7 @@ build_ctdb: stage: build tags: - docker + - shared script: - python script/autobuild.py samba-ctdb --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py ctdb --verbose --tail --testbase /tmp/samba-testbase @@ -78,6 +85,7 @@ build_others: stage: build tags: - docker + - shared script: - python script/autobuild.py ldb --verbose --tail --testbase /tmp/samba-testbase - python script/autobuild.py pidl --verbose --tail --testbase /tmp/samba-testbase From 9dc81b3e4e0aadc02dedffa01dd6e266fe3e8776 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 21 Mar 2018 07:02:42 +1300 Subject: [PATCH 16/24] autobuild: Split up the build further with samba-ad-dc-2 Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 9 +++++++++ .travis.yml | 1 + script/autobuild.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6846b2a782d..1e86d7ed9f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,15 @@ build_samba_ad_dc: # this one takes about 1 hours to finish - python script/autobuild.py samba-ad-dc --verbose --tail --testbase /tmp/samba-testbase +build_samba_ad_dc_2: + stage: build + tags: + - docker + - private + script: + # this one takes about 1 hours to finish + - python script/autobuild.py samba-ad-dc-2 --verbose --tail --testbase /tmp/samba-testbase + build_samba_none_env: stage: build tags: diff --git a/.travis.yml b/.travis.yml index 2ef0a0b1653..e9b6a6b2938 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ env: - TASK=samba-nt4 - TASK=samba-fileserver - TASK=samba-ad-dc + - TASK=samba-ad-dc-2 - TASK=ldb - TASK=tdb - TASK=talloc diff --git a/script/autobuild.py b/script/autobuild.py index 6e8abe77479..679e130ace8 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -36,6 +36,7 @@ "samba-test-only" : ".", "samba-none-env" : ".", "samba-ad-dc" : ".", + "samba-ad-dc-2" : ".", "samba-systemkrb5" : ".", "samba-nopython" : ".", "ldb" : "lib/ldb", @@ -60,6 +61,7 @@ "samba-static", "samba-none-env", "samba-ad-dc", + "samba-ad-dc-2", "samba-systemkrb5", "samba-nopython", "ldb", @@ -104,6 +106,9 @@ "--exclude-env=nt4_dc " "--exclude-env=nt4_member " "--exclude-env=ad_dc " + "--exclude-env=chgdcpass " + "--exclude-env=vampire_2000_dc " + "--exclude-env=fl2000dc " "--exclude-env=fileserver'", "text/plain"), ("install", "make install", "text/plain"), @@ -132,6 +137,20 @@ ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], + # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) + "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], + + # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) + "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], + "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ], From 4bf6291c684ed3b085818aaceea56a1095b1be6e Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 21 Mar 2018 07:24:23 +1300 Subject: [PATCH 17/24] gitlab: Run fileserver tests on "private" not "shared" This might make the delaywrite tests pass Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e86d7ed9f8..9c52057971c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ build_samba_fileserver: stage: build tags: - docker - - shared + - private script: # this one takes about 1 hours to finish - python script/autobuild.py samba-fileserver --verbose --tail --testbase /tmp/samba-testbase From 10d6100a73dd1337b3e24cfcce2f04e961996790 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 08:25:41 +1300 Subject: [PATCH 18/24] autobuild: Run all envs that depend on ad_dc in the ad_dc job Signed-off-by: Andrew Bartlett --- script/autobuild.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/script/autobuild.py b/script/autobuild.py index 679e130ace8..9f08abc00c4 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -106,6 +106,11 @@ "--exclude-env=nt4_dc " "--exclude-env=nt4_member " "--exclude-env=ad_dc " + "--exclude-env=fl2003dc " + "--exclude-env=fl2008r2dc " + "--exclude-env=ad_member " + "--exclude-env=ad_member_idmap_rid " + "--exclude-env=ad_member_idmap_ad " "--exclude-env=chgdcpass " "--exclude-env=vampire_2000_dc " "--exclude-env=fl2000dc " @@ -134,7 +139,13 @@ "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), ("make", "make -j", "text/plain"), - ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='" + "--include-env=ad_dc " + "--include-env=fl2003dc " + "--include-env=fl2008r2dc " + "--include-env=ad_member " + "--include-env=ad_member_idmap_rid " + "--include-env=ad_member_idmap_ad'", "text/plain"), ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")], # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long) From 8d240c8402f851bcbd58f15eb3bda6789824fa6b Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 12:02:16 +1300 Subject: [PATCH 19/24] selftest: Do not run raw.notify, smb2.oplock and raw.oplock twice These are slower tests that do not need to be run against the ad_dc configuration in particular. This saves time in the ad_dc job which needs to stay under 50mins to pass on travis-ci (and faster tests are better for everyone anyway). Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index 00e04318c7c..a4ded32195b 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -517,7 +517,8 @@ def plansmbtorture4testsuite(name, env, options, description=''): plansmbtorture4testsuite(t, env, '//$SERVER/tmp -U$USERNAME%$PASSWORD') elif t == "smb2.change_notify_disabled": plansmbtorture4testsuite(t, "simpleserver", '//$SERVER/tmp -U$USERNAME%$PASSWORD') - elif t == "smb2.notify": + elif t == "smb2.notify" or t == "raw.notify" or t == "smb2.oplock" or t == "raw.oplock": + # These tests are a little slower so don't duplicate them with ad_dc plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD --signing=required') elif t == "smb2.dosmode": plansmbtorture4testsuite(t, "simpleserver", '//$SERVER/dosmode -U$USERNAME%$PASSWORD') From 257f2925b9f2cad1fa902fc55f3618525987b850 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 12:21:52 +1300 Subject: [PATCH 20/24] selftest: Do not run krb5.kdc machine account test against ad_dc This code is already well tested against fl2008r2dc and just as per 8f1557a2c43e287c07723c16be78e1d858f4111d this test is slow and we can afford to be more selective here. Signed-off-by: Andrew Bartlett --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index adbca195c2a..e3c74900842 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -978,7 +978,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex "samba4.krb5.kdc with specified account") -for env in ["rodc", "promoted_dc", "ad_dc", "fl2000dc", "fl2008r2dc"]: +for env in ["rodc", "promoted_dc", "fl2000dc", "fl2008r2dc"]: if env == "rodc": # The machine account is cached at the RODC, as it is the local account extra_options = ['--option=torture:expect_rodc=true', '--option=torture:expect_cached_at_rodc=true'] From 9ff77e9ee0cfdc1fa57d5f0172fc5edbc18a7435 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 15:13:18 +1300 Subject: [PATCH 21/24] gitlab-ci: Split up build_samba_others and build_ctdb tasks These make too much output and the shared runners on GitLab CI object to sending more than 4MB of output. Signed-off-by: Andrew Bartlett --- .gitlab-ci.yml | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c52057971c..cde8341d0f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,17 +68,52 @@ build_samba_none_env: # this one takes about 1 hours to finish - python script/autobuild.py samba-none-env --verbose --tail --testbase /tmp/samba-testbase -build_samba_others: +build_samba_nopython: stage: build tags: - docker - shared script: - python script/autobuild.py samba-nopython --verbose --tail --testbase /tmp/samba-testbase + +build_samba_systemkrb5: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py samba-systemkrb5 --verbose --tail --testbase /tmp/samba-testbase + +build_samba_xc: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py samba-xc --verbose --tail --testbase /tmp/samba-testbase + +build_samba_o3: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py samba-o3 --verbose --tail --testbase /tmp/samba-testbase + +build_samba_libs: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py samba-libs --verbose --tail --testbase /tmp/samba-testbase + +build_samba_static: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py samba-static --verbose --tail --testbase /tmp/samba-testbase build_ctdb: @@ -88,6 +123,13 @@ build_ctdb: - shared script: - python script/autobuild.py samba-ctdb --verbose --tail --testbase /tmp/samba-testbase + +build_samba_ctdb: + stage: build + tags: + - docker + - shared + script: - python script/autobuild.py ctdb --verbose --tail --testbase /tmp/samba-testbase build_others: From 900573677cc0213294c2faed3c85f9c1c80356a7 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 16:25:59 +1300 Subject: [PATCH 22/24] selftest: Move slow raw_protocol test to chgdcpass environment The ad_dc environment is busy and we need to keep it under 50mins for travis CI, so run this on a different environment with a shorter runtime. Signed-off-by: Andrew Bartlett --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index e3c74900842..a8d18c5ded1 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -625,7 +625,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planoldpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.registry", extra_args=['-U"$USERNAME%$PASSWORD"']) planoldpythontestsuite("ad_dc_ntvfs", "samba.tests.dcerpc.dnsserver", extra_args=['-U"$USERNAME%$PASSWORD"']) planoldpythontestsuite("ad_dc", "samba.tests.dcerpc.dnsserver", extra_args=['-U"$USERNAME%$PASSWORD"']) -planoldpythontestsuite("ad_dc", "samba.tests.dcerpc.raw_protocol", extra_args=['-U"$USERNAME%$PASSWORD"']) +planoldpythontestsuite("chgdcpass", "samba.tests.dcerpc.raw_protocol", extra_args=['-U"$USERNAME%$PASSWORD"']) if have_heimdal_support: planoldpythontestsuite("ad_dc:local", "samba.tests.auth_log", extra_args=['-U"$USERNAME%$PASSWORD"'], environ={'CLIENT_IP': '127.0.0.11', From 2d14384fe13aa52155979ea196c741f53cbbff7a Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 16:45:05 +1300 Subject: [PATCH 23/24] selftest: Do not run *.lock tests against both nt4_dc and ad_dc This part of the protocol is not changed by being an AD DC. Signed-off-by: Andrew Bartlett --- source3/selftest/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index a4ded32195b..a9b7c209393 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -488,7 +488,8 @@ def plansmbtorture4testsuite(name, env, options, description=''): elif t == "smb2.lock": plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/aio -U$USERNAME%$PASSWORD', 'aio') plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD') - plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD') + elif t == "raw.lock" or t == "base.lock": + plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD') elif t == "raw.read": plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD') plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/aio -U$USERNAME%$PASSWORD', 'aio') From c5d55a5bdcb0e3eb4208ec1bd9b13a506fa08f1b Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 22 Mar 2018 16:49:53 +1300 Subject: [PATCH 24/24] selftest: Run net.api.become.dc against less roles This test is slower than many and need not be run five times. Signed-off-by: Andrew Bartlett --- source4/selftest/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index a8d18c5ded1..660a75fb927 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -263,6 +263,8 @@ def plansmbtorture4testsuite(name, env, options, modname=None): for bindoptions in ["connect", "krb5", "krb5,sign", "krb5,seal", "spnego", "spnego,sign", "spnego,seal"] + validate_list + ["padcheck", "bigendian", "bigendian,seal"]: echooptions = "--option=socket:testnonblock=True --option=torture:quick=yes -k yes" plansmbtorture4testsuite('rpc.echo', env, ["%s:$SERVER[%s]" % (transport, bindoptions), echooptions, '-U$USERNAME%$PASSWORD', '--workgroup=$DOMAIN'], "samba4.rpc.echo on %s with %s and %s" % (transport, bindoptions, echooptions)) + +for env in ["fl2000dc", "fl2008r2dc"]: plansmbtorture4testsuite("net.api.become.dc", env, '$SERVER[%s] -U$USERNAME%%$PASSWORD -W$DOMAIN' % validate) for bindoptions in ["sign", "seal"]: