[SCM] Samba Shared Repository - branch master updated
Matthieu Patou
mat at samba.org
Sat Mar 17 02:52:03 MDT 2012
The branch, master has been updated
via eeec0d9 upgrade provision didn't run findprovisionrange anymore
via 552d223 Rewrite findprovisionrange to use factorized functions
via 31cab61 Factorize code from findprovisionusnranges in order to solve bug #8504
from 8d00fe5 s3:gse: fix debug message in gse_get_server_auth_token()
http://gitweb.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit eeec0d925e3cc9bb33ed544815904f31c6c9b9ed
Author: Matthieu Patou <mat at matws.net>
Date: Sat Mar 17 00:19:40 2012 -0700
upgrade provision didn't run findprovisionrange anymore
Autobuild-User: Matthieu Patou <mat at samba.org>
Autobuild-Date: Sat Mar 17 09:51:46 CET 2012 on sn-devel-104
commit 552d223939495fe3809717ad29123ac3b7cd6e64
Author: Matthieu Patou <mat at matws.net>
Date: Sat Mar 17 00:19:09 2012 -0700
Rewrite findprovisionrange to use factorized functions
commit 31cab612fca8728fcdb486547846925d6e5f4d9e
Author: Matthieu Patou <mat at matws.net>
Date: Sat Mar 17 00:18:39 2012 -0700
Factorize code from findprovisionusnranges in order to solve bug #8504
-----------------------------------------------------------------------
Summary of changes:
source4/scripting/bin/findprovisionusnranges | 102 +-----------------
source4/scripting/bin/upgradeprovision | 21 +++--
source4/scripting/python/samba/upgradehelpers.py | 126 +++++++++++++++++++++-
3 files changed, 141 insertions(+), 108 deletions(-)
Changeset truncated at 500 lines:
diff --git a/source4/scripting/bin/findprovisionusnranges b/source4/scripting/bin/findprovisionusnranges
index 7ad88aa..ee9da3d 100755
--- a/source4/scripting/bin/findprovisionusnranges
+++ b/source4/scripting/bin/findprovisionusnranges
@@ -21,7 +21,6 @@
import sys
import optparse
-import tempfile
sys.path.insert(0, "bin/python")
from samba.credentials import DONT_USE_KERBEROS
@@ -31,10 +30,9 @@ import ldb
import samba.getopt as options
from samba import param
-from samba import _glue
-from samba.upgradehelpers import get_paths
+from samba.upgradehelpers import get_paths, print_provision_ranges, findprovisionrange
from samba.ndr import ndr_unpack
-from samba.dcerpc import drsblobs, misc
+from samba.dcerpc import misc
parser = optparse.OptionParser("provision [options]")
sambaopts = options.SambaOptions(parser)
@@ -54,10 +52,6 @@ paths = get_paths(param, smbconf=smbconf)
basedn="DC=" + lp.get("realm").replace(".",",DC=")
samdb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
-hash_id = {}
-ldif = ""
-nb_obj = 0
-
res = samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=["dsServiceName"])
invocation = None
@@ -75,100 +69,12 @@ else:
print "Unable to find attribute dsServiceName in rootDSE"
sys.exit(1)
-res = samdb.search(base=basedn, expression="objectClass=*",
- scope=ldb.SCOPE_SUBTREE,
- attrs=["replPropertyMetaData"],
- controls=["search_options:1:2"])
-
-for e in res:
- nb_obj = nb_obj + 1
- obj = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
- str(e["replPropertyMetaData"])).ctr
-
- for o in obj.array:
- # like a timestamp but with the resolution of 1 minute
- minutestamp =_glue.nttime2unix(o.originating_change_time)/60
- hash_ts = hash_id.get(str(o.originating_invocation_id))
- if hash_ts == None:
- ob = {}
- ob["min"] = o.originating_usn
- ob["max"] = o.originating_usn
- ob["num"] = 1
- ob["list"] = [str(e.dn)]
- hash_ts = {}
- else:
- ob = hash_ts.get(minutestamp)
- if ob == None:
- ob = {}
- ob["min"] = o.originating_usn
- ob["max"] = o.originating_usn
- ob["num"] = 1
- ob["list"] = [str(e.dn)]
- else:
- if ob["min"] > o.originating_usn:
- ob["min"] = o.originating_usn
- if ob["max"] < o.originating_usn:
- ob["max"] = o.originating_usn
- if not (str(e.dn) in ob["list"]):
- ob["num"] = ob["num"] + 1
- ob["list"].append(str(e.dn))
- hash_ts[minutestamp] = ob
- hash_id[str(o.originating_invocation_id)] = hash_ts
-
minobj = 5
+(hash_id, nb_obj) = findprovisionrange(samdb, basedn)
print "Here is a list of changes that modified more than %d objects in 1 minute." % minobj
print "Usually changes made by provision and upgradeprovision are those who affect a couple"\
" of hundred of objects or more"
print "Total number of objects: %d" % nb_obj
print
-for id in hash_id:
- hash_ts = hash_id[id]
- sorted_keys = []
- sorted_keys.extend(hash_ts.keys())
- sorted_keys.sort()
-
- kept_record = []
- for k in sorted_keys:
- obj = hash_ts[k]
- if obj["num"] > minobj:
- dt = _glue.nttime2string(_glue.unix2nttime(k*60))
- print "%s # of modification: %d \tmin: %d max: %d" % (dt , obj["num"],
- obj["min"],
- obj["max"])
- if hash_ts[k]["num"] > 600:
- kept_record.append(k)
-
- # Let's try to concatenate consecutive block if they are in the almost same minutestamp
- for i in range(0, len(kept_record)):
- if i != 0:
- key1 = kept_record[i]
- key2 = kept_record[i-1]
- if key1 - key2 == 1:
- # previous record is just 1 minute away from current
- if int(hash_ts[key1]["min"]) == int(hash_ts[key2]["max"]) + 1:
- # Copy the highest USN in the previous record
- # and mark the current as skipped
- hash_ts[key2]["max"] = hash_ts[key1]["max"]
- hash_ts[key1]["skipped"] = True
-
- for k in kept_record:
- obj = hash_ts[k]
- if obj.get("skipped") == None:
- ldif = "%slastProvisionUSN: %d-%d;%s\n" % (ldif, obj["min"],
- obj["max"], id)
-
-if ldif != "":
- dest = opts.storedir
- if dest == None:
- dest = "/tmp"
-
- file = tempfile.mktemp(dir=dest, prefix="usnprov", suffix=".ldif")
- print
- print "To track the USNs modified/created by provision and upgrade proivsion,"
- print " the following ranges are proposed to be added to your provision sam.ldb: \n%s" % ldif
- print "We recommend to review them, and if it's correct to integrate the following ldif: %s in your sam.ldb" % file
- print "You can load this file like this: ldbadd -H %s %s\n"%(str(paths.samdb),file)
- ldif = "dn: @PROVISION\nprovisionnerID: %s\n%s" % (invocation, ldif)
- open(file,'w').write(ldif)
-
+print_provision_ranges(hash_id, minobj, opts.storedir, str(paths.samdb), invocation)
diff --git a/source4/scripting/bin/upgradeprovision b/source4/scripting/bin/upgradeprovision
index 41edd07..e578c14 100755
--- a/source4/scripting/bin/upgradeprovision
+++ b/source4/scripting/bin/upgradeprovision
@@ -53,7 +53,7 @@ from samba.schema import get_linked_attributes, Schema, get_schema_descriptor
from samba.dcerpc import security, drsblobs
from samba.ndr import ndr_unpack
from samba.upgradehelpers import (dn_sort, get_paths, newprovision,
- get_ldbs,
+ get_ldbs, findprovisionrange,
usn_in_range, identic_rename, get_diff_sddls,
update_secrets, CHANGE, ERROR, SIMPLE,
CHANGEALL, GUESS, CHANGESD, PROVISION,
@@ -62,7 +62,8 @@ from samba.upgradehelpers import (dn_sort, get_paths, newprovision,
update_machine_account_password,
search_constructed_attrs_stored,
int64range2str, update_dns_account_password,
- increment_calculated_keyversion_number)
+ increment_calculated_keyversion_number,
+ print_provision_ranges)
from samba.xattr import copytree_with_xattrs
replace=2**FLAG_MOD_REPLACE
@@ -1698,12 +1699,16 @@ if __name__ == '__main__':
message(SIMPLE, "Your provision lacks provision range information")
if confirm("Do you want to run findprovisionusnranges to try to find them ?", False):
ldbs.groupedRollback()
- os.system("%s %s %s %s %s" % (os.path.join(os.path.dirname(sys.argv[0]),
- "findprovisionusnranges"),
- "--storedir",
- paths.private_dir,
- "-s",
- smbconf))
+ minobj = 5
+ (hash_id, nb_obj) = findprovisionrange(ldbs.sam, ldb.Dn(ldbs.sam, str(names.rootdn)))
+ message(SIMPLE, "Here is a list of changes that modified more than %d objects in 1 minute." % minobj)
+ message(SIMPLE, "Usually changes made by provision and upgradeprovision are those who affect a couple"\
+ " of hundred of objects or more")
+ message(SIMPLE, "Total number of objects: %d" % nb_obj)
+ message(SIMPLE, "")
+
+ print_provision_ranges(hash_id, minobj, None, str(paths.samdb), str(names.invocation))
+
message(SIMPLE, "Once you applied/adapted the change(s) please restart the upgradeprovision script")
sys.exit(0)
diff --git a/source4/scripting/python/samba/upgradehelpers.py b/source4/scripting/python/samba/upgradehelpers.py
index 3507661..d049fa2 100644
--- a/source4/scripting/python/samba/upgradehelpers.py
+++ b/source4/scripting/python/samba/upgradehelpers.py
@@ -1,5 +1,5 @@
# Helpers for provision stuff
-# Copyright (C) Matthieu Patou <mat at matws.net> 2009-2010
+# Copyright (C) Matthieu Patou <mat at matws.net> 2009-2012
#
# Based on provision a Samba4 server by
# Copyright (C) Jelmer Vernooij <jelmer at samba.org> 2007-2008
@@ -33,9 +33,12 @@ from samba.provision import (provision_paths_from_lp,
getpolicypath, set_gpos_acl, create_gpo_struct,
FILL_FULL, provision, ProvisioningError,
setsysvolacl, secretsdb_self_join)
-from samba.dcerpc import xattr
+from samba.dcerpc import xattr, drsblobs
from samba.dcerpc.misc import SEC_CHAN_BDC
+from samba.ndr import ndr_unpack
from samba.samdb import SamDB
+from samba import _glue
+import tempfile
# All the ldb related to registry are commented because the path for them is
# relative in the provisionPath object
@@ -833,6 +836,125 @@ def search_constructed_attrs_stored(samdb, rootdn, attrs):
return hashAtt
+def findprovisionrange(samdb, basedn):
+ """ Find ranges of usn grouped by invocation id and then by timestamp
+ rouned at 1 minute
+
+ :param samdb: An LDB object pointing to the samdb
+ :param basedn: The DN of the forest
+
+ :return: A two level dictionary with invoication id as the
+ first level, timestamp as the second one and then
+ max, min, and number as subkeys, representing respectivily
+ the maximum usn for the range, the minimum usn and the number
+ of object with usn in this range.
+ """
+ nb_obj = 0
+ hash_id = {}
+
+ res = samdb.search(base=basedn, expression="objectClass=*",
+ scope=ldb.SCOPE_SUBTREE,
+ attrs=["replPropertyMetaData"],
+ controls=["search_options:1:2"])
+
+ for e in res:
+ nb_obj = nb_obj + 1
+ obj = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
+ str(e["replPropertyMetaData"])).ctr
+
+ for o in obj.array:
+ # like a timestamp but with the resolution of 1 minute
+ minutestamp =_glue.nttime2unix(o.originating_change_time)/60
+ hash_ts = hash_id.get(str(o.originating_invocation_id))
+
+ if hash_ts == None:
+ ob = {}
+ ob["min"] = o.originating_usn
+ ob["max"] = o.originating_usn
+ ob["num"] = 1
+ ob["list"] = [str(e.dn)]
+ hash_ts = {}
+ else:
+ ob = hash_ts.get(minutestamp)
+ if ob == None:
+ ob = {}
+ ob["min"] = o.originating_usn
+ ob["max"] = o.originating_usn
+ ob["num"] = 1
+ ob["list"] = [str(e.dn)]
+ else:
+ if ob["min"] > o.originating_usn:
+ ob["min"] = o.originating_usn
+ if ob["max"] < o.originating_usn:
+ ob["max"] = o.originating_usn
+ if not (str(e.dn) in ob["list"]):
+ ob["num"] = ob["num"] + 1
+ ob["list"].append(str(e.dn))
+ hash_ts[minutestamp] = ob
+ hash_id[str(o.originating_invocation_id)] = hash_ts
+
+ return (hash_id, nb_obj)
+
+def print_provision_ranges(dic, limit_print, dest, samdb_path, invocationid):
+ """ print the differents ranges passed as parameter
+
+ :param dic: A dictionnary as returned by findprovisionrange
+ :param limit_print: minimum number of object in a range in order to print it
+ :param dest: Destination directory
+ :param samdb_path: Path to the sam.ldb file
+ :param invoicationid: Invocation ID for the current provision
+ """
+ ldif = ""
+
+ for id in dic:
+ hash_ts = dic[id]
+ sorted_keys = []
+ sorted_keys.extend(hash_ts.keys())
+ sorted_keys.sort()
+
+ kept_record = []
+ for k in sorted_keys:
+ obj = hash_ts[k]
+ if obj["num"] > limit_print:
+ dt = _glue.nttime2string(_glue.unix2nttime(k*60))
+ print "%s # of modification: %d \tmin: %d max: %d" % (dt , obj["num"],
+ obj["min"],
+ obj["max"])
+ if hash_ts[k]["num"] > 600:
+ kept_record.append(k)
+
+ # Let's try to concatenate consecutive block if they are in the almost same minutestamp
+ for i in range(0, len(kept_record)):
+ if i != 0:
+ key1 = kept_record[i]
+ key2 = kept_record[i-1]
+ if key1 - key2 == 1:
+ # previous record is just 1 minute away from current
+ if int(hash_ts[key1]["min"]) == int(hash_ts[key2]["max"]) + 1:
+ # Copy the highest USN in the previous record
+ # and mark the current as skipped
+ hash_ts[key2]["max"] = hash_ts[key1]["max"]
+ hash_ts[key1]["skipped"] = True
+
+ for k in kept_record:
+ obj = hash_ts[k]
+ if obj.get("skipped") == None:
+ ldif = "%slastProvisionUSN: %d-%d;%s\n" % (ldif, obj["min"],
+ obj["max"], id)
+
+ if ldif != "":
+ if dest == None:
+ dest = "/tmp"
+
+ file = tempfile.mktemp(dir=dest, prefix="usnprov", suffix=".ldif")
+ print
+ print "To track the USNs modified/created by provision and upgrade proivsion,"
+ print " the following ranges are proposed to be added to your provision sam.ldb: \n%s" % ldif
+ print "We recommend to review them, and if it's correct to integrate the following ldif: %s in your sam.ldb" % file
+ print "You can load this file like this: ldbadd -H %s %s\n"%(str(samdb_path),file)
+ ldif = "dn: @PROVISION\nprovisionnerID: %s\n%s" % (invocationid, ldif)
+ open(file,'w').write(ldif)
+
def int64range2str(value):
"""Display the int64 range stored in value as xxx-yyy
--
Samba Shared Repository
More information about the samba-cvs
mailing list