From 407b2baffb9ff13b4e967e5662faa42bbfb29482 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 01/18] s4:scripting/python: always treat the highwatermark as opaque (bug #9508) Signed-off-by: Stefan Metzmacher --- source4/scripting/devel/getncchanges | 2 +- source4/scripting/devel/repl_cleartext_pwd.py | 2 +- source4/scripting/python/samba/drs_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source4/scripting/devel/getncchanges b/source4/scripting/devel/getncchanges index 736e3db..d401c82 100755 --- a/source4/scripting/devel/getncchanges +++ b/source4/scripting/devel/getncchanges @@ -177,4 +177,4 @@ if __name__ == "__main__": nb_iter += 1 if ctr.more_data == 0 or opts.nb_iter == nb_iter: break - req8.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req8.highwatermark = ctr.new_highwatermark diff --git a/source4/scripting/devel/repl_cleartext_pwd.py b/source4/scripting/devel/repl_cleartext_pwd.py index 5eebc86..9637d88 100755 --- a/source4/scripting/devel/repl_cleartext_pwd.py +++ b/source4/scripting/devel/repl_cleartext_pwd.py @@ -406,4 +406,4 @@ if __name__ == "__main__": print "# up to usn[%d]" % (ctr.new_highwatermark.highest_usn) break print "# up to tmp_usn[%d]" % (ctr.new_highwatermark.highest_usn) - req8.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req8.highwatermark = ctr.new_highwatermark diff --git a/source4/scripting/python/samba/drs_utils.py b/source4/scripting/python/samba/drs_utils.py index 481eec2..6e2cfea 100644 --- a/source4/scripting/python/samba/drs_utils.py +++ b/source4/scripting/python/samba/drs_utils.py @@ -252,4 +252,4 @@ class drs_Replicate(object): schema=schema, req_level=req_level, req=req) if ctr.more_data == 0: break - req.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req.highwatermark = ctr.new_highwatermark -- 1.7.9.5 From 5229d2912aed6b5fb2fedcd3cbae8211f39702bd Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 02/18] s4:dsdb/repl_meta_data: always treat the highwatermark as opaque (bug #9508) Signed-off-by: Stefan Metzmacher --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 3ac1e6a..868484d 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4652,7 +4652,6 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a ZERO_STRUCT(nrf); nrf.version = 1; nrf.ctr.ctr1 = *ar->objs->source_dsa; - nrf.ctr.ctr1.highwatermark.highest_usn = nrf.ctr.ctr1.highwatermark.tmp_highest_usn; /* * first see if we already have a repsFrom value for the current source dsa -- 1.7.9.5 From 2b54aa94ee4efd10418ab1791bff050a587852ef Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:29:04 +0100 Subject: [PATCH 03/18] s4:dsdb/repl_meta_data: store the last results and timestamps in the repsFrom Signed-off-by: Stefan Metzmacher --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 868484d..6a69f7d 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4652,6 +4652,9 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a ZERO_STRUCT(nrf); nrf.version = 1; nrf.ctr.ctr1 = *ar->objs->source_dsa; + nrf.ctr.ctr1.last_attempt = now; + nrf.ctr.ctr1.last_success = now; + nrf.ctr.ctr1.result_last_attempt = WERR_OK; /* * first see if we already have a repsFrom value for the current source dsa -- 1.7.9.5 From c4080f7d3ea0e44f1d4e8569e892570b5d7e5acc Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 20 Dec 2012 15:46:05 +0100 Subject: [PATCH 04/18] s4:dsdb/repl_meta_data: also update the last_sync_success in replUpToDateVector This matches Windows 2008R2 and Windows 2012. Signed-off-by: Stefan Metzmacher --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 6a69f7d..b7d9248 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4563,12 +4563,8 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a found = true; - /* - * we update only the highest_usn and not the latest_sync_success time, - * because the last success stands for direct replication - */ if (ruv->cursors[i].highest_usn > nuv.ctr.ctr2.cursors[j].highest_usn) { - nuv.ctr.ctr2.cursors[j].highest_usn = ruv->cursors[i].highest_usn; + nuv.ctr.ctr2.cursors[j] = ruv->cursors[i]; } break; } -- 1.7.9.5 From 96fbe54c711cd9f139932f3e783a9ef0f5345a44 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 05/18] s4:dsdb/repl_meta_data: don't merge highwatermark and uptodatevector (bug #9508) We should not do any magic regarding the highwatermark we got from the source dsa. We need to treat it as opaque and not try to be smart and merge it into the uptodatevector. Signed-off-by: Stefan Metzmacher --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 39 +---------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index b7d9248..30b2a42 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4529,7 +4529,7 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a * * plus optional values from our old vector and the one from the source_dsa */ - nuv.ctr.ctr2.count = 1 + ouv.ctr.ctr2.count; + nuv.ctr.ctr2.count = ouv.ctr.ctr2.count; if (ruv) nuv.ctr.ctr2.count += ruv->count; nuv.ctr.ctr2.cursors = talloc_array(ar, struct drsuapi_DsReplicaCursor2, @@ -4577,43 +4577,6 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a } /* - * merge in the current highwatermark for the source_dsa - */ - found = false; - for (j=0; j < ni; j++) { - if (!GUID_equal(&ar->objs->source_dsa->source_dsa_invocation_id, - &nuv.ctr.ctr2.cursors[j].source_dsa_invocation_id)) { - continue; - } - - found = true; - - /* - * here we update the highest_usn and last_sync_success time - * because we're directly replicating from the source_dsa - * - * and use the tmp_highest_usn because this is what we have just applied - * to our ldb - */ - nuv.ctr.ctr2.cursors[j].highest_usn = ar->objs->source_dsa->highwatermark.tmp_highest_usn; - nuv.ctr.ctr2.cursors[j].last_sync_success = now; - break; - } - if (!found) { - /* - * here we update the highest_usn and last_sync_success time - * because we're directly replicating from the source_dsa - * - * and use the tmp_highest_usn because this is what we have just applied - * to our ldb - */ - nuv.ctr.ctr2.cursors[ni].source_dsa_invocation_id= ar->objs->source_dsa->source_dsa_invocation_id; - nuv.ctr.ctr2.cursors[ni].highest_usn = ar->objs->source_dsa->highwatermark.tmp_highest_usn; - nuv.ctr.ctr2.cursors[ni].last_sync_success = now; - ni++; - } - - /* * finally correct the size of the cursors array */ nuv.ctr.ctr2.count = ni; -- 1.7.9.5 From 852d2c2bffb97bbde878a2970994ff811381ebde Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 12:47:43 +0100 Subject: [PATCH 06/18] s4:dsdb/common: use LDB_SEQ_HIGHEST_SEQ for our entry in the uptodatevector We should use the global highestCommittedUSN, not the per partition value. This matches a Windows 2008R2 and 2012 server. Signed-off-by: Stefan Metzmacher --- source4/dsdb/common/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c index 4543003..894fd3c 100644 --- a/source4/dsdb/common/util.c +++ b/source4/dsdb/common/util.c @@ -3487,7 +3487,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m const struct ldb_val *ouv_value; unsigned int i; int ret; - uint64_t highest_usn; + uint64_t highest_usn = 0; const struct GUID *our_invocation_id; struct timeval now = timeval_current(); @@ -3530,7 +3530,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m return ldb_operr(samdb); } - ret = dsdb_load_partition_usn(samdb, dn, &highest_usn, NULL); + ret = ldb_sequence_number(samdb, LDB_SEQ_HIGHEST_SEQ, &highest_usn); if (ret != LDB_SUCCESS) { /* nothing to add - this can happen after a vampire */ TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); -- 1.7.9.5 From bf522b3e1e7cefc9cd3b1e7f6d73299fede35a52 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:33:13 +0100 Subject: [PATCH 07/18] s4:dsdb/common: use 01.01.1970 as last_sync_success for our entry in the uptodatevector This matches a Windows 2008R2 and 2012 server. Signed-off-by: Stefan Metzmacher --- source4/dsdb/common/util.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c index 894fd3c..2b96bd4 100644 --- a/source4/dsdb/common/util.c +++ b/source4/dsdb/common/util.c @@ -3489,7 +3489,8 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m int ret; uint64_t highest_usn = 0; const struct GUID *our_invocation_id; - struct timeval now = timeval_current(); + static const struct timeval tv1970; + NTTIME nt1970 = timeval_to_nttime(&tv1970); ret = ldb_search(samdb, mem_ctx, &r, dn, LDB_SCOPE_BASE, attrs, NULL); if (ret != LDB_SUCCESS) { @@ -3540,7 +3541,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m for (i=0; i<*count; i++) { if (GUID_equal(our_invocation_id, &(*cursors)[i].source_dsa_invocation_id)) { (*cursors)[i].highest_usn = highest_usn; - (*cursors)[i].last_sync_success = timeval_to_nttime(&now); + (*cursors)[i].last_sync_success = nt1970; TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); return LDB_SUCCESS; } @@ -3553,7 +3554,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m (*cursors)[*count].source_dsa_invocation_id = *our_invocation_id; (*cursors)[*count].highest_usn = highest_usn; - (*cursors)[*count].last_sync_success = timeval_to_nttime(&now); + (*cursors)[*count].last_sync_success = nt1970; (*count)++; TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); -- 1.7.9.5 From 08b1ba0a247490a7f8226bbee08ed990a2b889e0 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:31:28 +0100 Subject: [PATCH 08/18] s4:dsdb/drepl: update the source_dsa_obj/invocation_id in repsFrom The highwatermark is relative to the source_dsa_invocation_id. Signed-off-by: Stefan Metzmacher --- source4/dsdb/repl/drepl_out_helpers.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source4/dsdb/repl/drepl_out_helpers.c b/source4/dsdb/repl/drepl_out_helpers.c index 16825d4..57205a8 100644 --- a/source4/dsdb/repl/drepl_out_helpers.c +++ b/source4/dsdb/repl/drepl_out_helpers.c @@ -627,6 +627,8 @@ static void dreplsrv_op_pull_source_apply_changes_trigger(struct tevent_req *req first_object = ctr1->first_object; linked_attributes_count = 0; linked_attributes = NULL; + rf1.source_dsa_obj_guid = ctr1->source_dsa_guid; + rf1.source_dsa_invocation_id = ctr1->source_dsa_invocation_id; rf1.highwatermark = ctr1->new_highwatermark; uptodateness_vector = NULL; /* TODO: map it */ more_data = ctr1->more_data; @@ -637,6 +639,8 @@ static void dreplsrv_op_pull_source_apply_changes_trigger(struct tevent_req *req first_object = ctr6->first_object; linked_attributes_count = ctr6->linked_attributes_count; linked_attributes = ctr6->linked_attributes; + rf1.source_dsa_obj_guid = ctr6->source_dsa_guid; + rf1.source_dsa_invocation_id = ctr6->source_dsa_invocation_id; rf1.highwatermark = ctr6->new_highwatermark; uptodateness_vector = ctr6->uptodateness_vector; more_data = ctr6->more_data; -- 1.7.9.5 From 18f9de5e3ebde5e306412deb5ab549ab7efca419 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 14:08:56 +0100 Subject: [PATCH 09/18] s4:drsuapi: move struct drsuapi_getncchanges_state to the top of getncchanges.c Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 09406d6..00988fc 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -37,6 +37,23 @@ #include "auth/session.h" #include "dsdb/common/util.h" +/* state of a partially completed getncchanges call */ +struct drsuapi_getncchanges_state { + struct GUID *guids; + uint32_t num_records; + uint32_t num_processed; + struct ldb_dn *ncRoot_dn; + bool is_schema_nc; + uint64_t min_usn; + uint64_t highest_usn; + struct ldb_dn *last_dn; + struct drsuapi_DsReplicaLinkedAttribute *la_list; + uint32_t la_count; + bool la_sorted; + uint32_t la_idx; + struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; +}; + /* build a DsReplicaObjectIdentifier from a ldb msg */ @@ -1147,23 +1164,6 @@ static WERROR getncchanges_change_master(struct drsuapi_bind_state *b_state, return WERR_OK; } -/* state of a partially completed getncchanges call */ -struct drsuapi_getncchanges_state { - struct GUID *guids; - uint32_t num_records; - uint32_t num_processed; - struct ldb_dn *ncRoot_dn; - bool is_schema_nc; - uint64_t min_usn; - uint64_t highest_usn; - struct ldb_dn *last_dn; - struct drsuapi_DsReplicaLinkedAttribute *la_list; - uint32_t la_count; - bool la_sorted; - uint32_t la_idx; - struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; -}; - /* see if this getncchanges request includes a request to reveal secret information */ -- 1.7.9.5 From 07780ecc984e720e96bdde97fb2c95291b0ed0b8 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 13:48:01 +0100 Subject: [PATCH 10/18] s4:drsuapi: remove unused 'highest_usn' from drsuapi_getncchanges_state Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 00988fc..b7b488c 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,7 +45,6 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; - uint64_t highest_usn; struct ldb_dn *last_dn; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; @@ -1876,9 +1875,6 @@ allowed: if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; } - if (uSN > getnc_state->highest_usn) { - getnc_state->highest_usn = uSN; - } if (obj->meta_data_ctr == NULL) { DEBUG(8,(__location__ ": getncchanges skipping send of object %s\n", -- 1.7.9.5 From 6a8b5dbd10cf030ce5585c4b40711b7653f32b91 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 12:44:43 +0100 Subject: [PATCH 11/18] s4:drsuapi: avoid a ldb_dn_copy() and use talloc_move() instead Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index b7b488c..7ed47f1 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -1889,11 +1889,11 @@ allowed: *currentObject = obj; currentObject = &obj->next_object; - talloc_free(getnc_state->last_dn); - getnc_state->last_dn = ldb_dn_copy(getnc_state, msg->dn); - DEBUG(8,(__location__ ": replicating object %s\n", ldb_dn_get_linearized(msg->dn))); + talloc_free(getnc_state->last_dn); + getnc_state->last_dn = talloc_move(getnc_state, &msg->dn); + talloc_free(msg_res); talloc_free(msg_dn); } -- 1.7.9.5 From 91bd3b53a1a95edf4935987fd48cc1db6ed44cda Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 16:34:25 +0100 Subject: [PATCH 12/18] s4:drsuapi: always use the current uptodateness_vector Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 7ed47f1..b3ceb50 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -50,7 +50,6 @@ struct drsuapi_getncchanges_state { uint32_t la_count; bool la_sorted; uint32_t la_idx; - struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; }; /* @@ -1730,14 +1729,6 @@ allowed: site_res_cmp_dn_usn_order); } - getnc_state->uptodateness_vector = talloc_steal(getnc_state, req10->uptodateness_vector); - if (getnc_state->uptodateness_vector) { - /* make sure its sorted */ - TYPESAFE_QSORT(getnc_state->uptodateness_vector->cursors, - getnc_state->uptodateness_vector->count, - drsuapi_DsReplicaCursor_compare); - } - for (i=0; i < getnc_state->num_records; i++) { getnc_state->guids[i] = changes[i].guid; if (GUID_all_zero(&getnc_state->guids[i])) { @@ -1751,6 +1742,13 @@ allowed: talloc_free(changes); } + if (req10->uptodateness_vector) { + /* make sure its sorted */ + TYPESAFE_QSORT(req10->uptodateness_vector->cursors, + req10->uptodateness_vector->count, + drsuapi_DsReplicaCursor_compare); + } + /* Prefix mapping */ schema = dsdb_get_schema(sam_ctx, mem_ctx); if (!schema) { @@ -1852,7 +1850,7 @@ allowed: schema, &session_key, getnc_state->min_usn, req10->replica_flags, req10->partial_attribute_set, - getnc_state->uptodateness_vector, + req10->uptodateness_vector, req10->extended_op, max_wait_reached); if (!W_ERROR_IS_OK(werr)) { @@ -1866,7 +1864,7 @@ allowed: msg, &getnc_state->la_list, &getnc_state->la_count, - getnc_state->uptodateness_vector); + req10->uptodateness_vector); if (!W_ERROR_IS_OK(werr)) { return werr; } -- 1.7.9.5 From 492493d2af8d54f07115cab9c7f50b93490f65ec Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 11:13:43 +0100 Subject: [PATCH 13/18] s4:drsuapi: add drsuapi_DsReplicaHighWaterMark_cmp() Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index b3ceb50..1f597c6 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -52,6 +52,26 @@ struct drsuapi_getncchanges_state { uint32_t la_idx; }; +static int drsuapi_DsReplicaHighWaterMark_cmp(const struct drsuapi_DsReplicaHighWaterMark *h1, + const struct drsuapi_DsReplicaHighWaterMark *h2) +{ + if (h1->highest_usn < h2->highest_usn) { + return -1; + } else if (h1->highest_usn > h2->highest_usn) { + return 1; + } else if (h1->tmp_highest_usn < h2->tmp_highest_usn) { + return -1; + } else if (h1->tmp_highest_usn > h2->tmp_highest_usn) { + return 1; + } else if (h1->reserved_usn < h2->reserved_usn) { + return -1; + } else if (h1->reserved_usn > h2->reserved_usn) { + return 1; + } + + return 0; +} + /* build a DsReplicaObjectIdentifier from a ldb msg */ -- 1.7.9.5 From ae6ced8feed14ead42675f6b204467d7512a4f6e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 11:30:26 +0100 Subject: [PATCH 14/18] s4:drsuapi: make sure we never return the same highwatermark twice in a replication cycle (bug #9508) If the highwatermark given by the client is not the one we expect, we need to start a new replication cycle. Otherwise the destination dsa skips objects and linked attribute values. Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 1f597c6..e06aeed 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,6 +45,7 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; + struct drsuapi_DsReplicaHighWaterMark last_hwm; struct ldb_dn *last_dn; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; @@ -1620,6 +1621,20 @@ allowed: } } + if (getnc_state) { + ret = drsuapi_DsReplicaHighWaterMark_cmp(&getnc_state->last_hwm, + &req10->highwatermark); + if (ret != 0) { + DEBUG(0,(__location__ ": DsGetNCChanges 2nd replication " + "on DN %s %s highwatermark (last_dn %s)\n", + ldb_dn_get_linearized(getnc_state->ncRoot_dn), + (ret > 0) ? "older" : "newer", + ldb_dn_get_linearized(getnc_state->last_dn))); + talloc_free(getnc_state); + getnc_state = NULL; + } + } + if (getnc_state == NULL) { getnc_state = talloc_zero(b_state, struct drsuapi_getncchanges_state); if (getnc_state == NULL) { @@ -1892,6 +1907,7 @@ allowed: uSN = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1); if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; + r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; } if (obj->meta_data_ctr == NULL) { @@ -1993,6 +2009,7 @@ allowed: r->out.ctr->ctr6.uptodateness_vector = talloc(mem_ctx, struct drsuapi_DsReplicaCursor2CtrEx); r->out.ctr->ctr6.new_highwatermark.highest_usn = r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn; + r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, r->out.ctr->ctr6.uptodateness_vector); @@ -2002,6 +2019,25 @@ allowed: talloc_free(getnc_state); b_state->getncchanges_state = NULL; + } else { + ret = drsuapi_DsReplicaHighWaterMark_cmp(&r->out.ctr->ctr6.old_highwatermark, + &r->out.ctr->ctr6.new_highwatermark); + if (ret == 0) { + /* + * We need to make sure that we never return the + * same highwatermark within the same replication + * cycle more than once. Otherwise we cannot detect + * when the client uses an unexptected highwatermark. + * + * This is a HACK which is needed because our + * object ordering is wrong and set tmp_highest_usn + * to a value that is higher than what we already + * sent to the client (destination dsa). + */ + r->out.ctr->ctr6.new_highwatermark.reserved_usn += 1; + } + + getnc_state->last_hwm = r->out.ctr->ctr6.new_highwatermark; } if (req10->extended_op != DRSUAPI_EXOP_NONE) { -- 1.7.9.5 From 82621c2927b7f467ba90a60d009411d3eaeb2fa0 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 13:40:33 +0100 Subject: [PATCH 15/18] s4:drsuapi: check the source_dsa_invocation_id (bug #9508) The given highwatermark is only valid relative to the specified source_dsa_invocation_id. Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index e06aeed..10965a3 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -1483,12 +1483,15 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_ time_t start = time(NULL); bool max_wait_reached = false; bool has_get_all_changes = false; + struct GUID invocation_id; DCESRV_PULL_HANDLE_WERR(h, r->in.bind_handle, DRSUAPI_BIND_HANDLE); b_state = h->data; sam_ctx = b_state->sam_ctx_system?b_state->sam_ctx_system:b_state->sam_ctx; + invocation_id = *(samdb_ntds_invocation_id(sam_ctx)); + *r->out.level_out = 6; /* TODO: linked attributes*/ r->out.ctr->ctr6.linked_attributes_count = 0; @@ -1606,6 +1609,18 @@ allowed: req10->uptodateness_vector = NULL; } + if (GUID_all_zero(&req10->source_dsa_invocation_id)) { + req10->source_dsa_invocation_id = invocation_id; + } + + if (!GUID_equal(&req10->source_dsa_invocation_id, &invocation_id)) { + /* + * The given highwatermark is only valid relative to the + * specified source_dsa_invocation_id. + */ + ZERO_STRUCT(req10->highwatermark); + } + getnc_state = b_state->getncchanges_state; /* see if a previous replication has been abandoned */ -- 1.7.9.5 From 89dd3c6c0f41216815971d52083ee0397264635d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:59:20 +0100 Subject: [PATCH 16/18] s4:drsuapi: make sure we report the meta data from the cycle start (bug #9508) We should build the final highwatermark and uptodatevector of a replication cycle at the start of the cycle. Before we search for the currently missing objects. Otherwise we risk that some objects get lost. Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 47 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 10965a3..333ffc2 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,8 +45,11 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; + uint64_t max_usn; struct drsuapi_DsReplicaHighWaterMark last_hwm; struct ldb_dn *last_dn; + struct drsuapi_DsReplicaHighWaterMark final_hwm; + struct drsuapi_DsReplicaCursor2CtrEx *final_udv; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; bool la_sorted; @@ -1740,6 +1743,18 @@ allowed: extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter"); getnc_state->min_usn = req10->highwatermark.highest_usn; + getnc_state->max_usn = getnc_state->min_usn; + + getnc_state->final_udv = talloc_zero(getnc_state, + struct drsuapi_DsReplicaCursor2CtrEx); + if (getnc_state->final_udv == NULL) { + return WERR_NOMEM; + } + werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, + getnc_state->final_udv); + if (!W_ERROR_IS_OK(werr)) { + return werr; + } if (req10->extended_op == DRSUAPI_EXOP_NONE) { werr = getncchanges_collect_objects(b_state, mem_ctx, req10, @@ -1767,6 +1782,10 @@ allowed: changes[i].dn = search_res->msgs[i]->dn; changes[i].guid = samdb_result_guid(search_res->msgs[i], "objectGUID"); changes[i].usn = ldb_msg_find_attr_as_uint64(search_res->msgs[i], "uSNChanged", 0); + + if (changes[i].usn > getnc_state->max_usn) { + getnc_state->max_usn = changes[i].usn; + } } if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) { @@ -1788,6 +1807,10 @@ allowed: } } + getnc_state->final_hwm.tmp_highest_usn = getnc_state->max_usn; + getnc_state->final_hwm.reserved_usn = 0; + getnc_state->final_hwm.highest_usn = getnc_state->max_usn; + talloc_free(search_res); talloc_free(changes); } @@ -1920,6 +1943,18 @@ allowed: } uSN = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1); + if (uSN > getnc_state->max_usn) { + /* + * Only report the max_usn we had at the start + * of the replication cycle. + * + * If this object has changed lately we better + * let the destination dsa refetch the chance. + * This is better than the risk of loosing some + * objects or linked attributes. + */ + uSN = 0; + } if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; @@ -2022,15 +2057,9 @@ allowed: if (!r->out.ctr->ctr6.more_data) { talloc_steal(mem_ctx, getnc_state->la_list); - r->out.ctr->ctr6.uptodateness_vector = talloc(mem_ctx, struct drsuapi_DsReplicaCursor2CtrEx); - r->out.ctr->ctr6.new_highwatermark.highest_usn = r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn; - r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; - - werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, - r->out.ctr->ctr6.uptodateness_vector); - if (!W_ERROR_IS_OK(werr)) { - return werr; - } + r->out.ctr->ctr6.new_highwatermark = getnc_state->final_hwm; + r->out.ctr->ctr6.uptodateness_vector = talloc_move(mem_ctx, + &getnc_state->final_udv); talloc_free(getnc_state); b_state->getncchanges_state = NULL; -- 1.7.9.5 From 646b19d8d614a39c573865ce0e0f5924e6023706 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 15:16:28 +0100 Subject: [PATCH 17/18] s4:drsuapi: make use of LDB_TYPESAFE_QSORT() and pass getnc_state Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 333ffc2..729b23d 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -686,8 +686,9 @@ struct drsuapi_changed_objects { /* sort the objects we send by tree order */ -static int site_res_cmp_parent_order(struct drsuapi_changed_objects *m1, - struct drsuapi_changed_objects *m2) +static int site_res_cmp_anc_order(struct drsuapi_changed_objects *m1, + struct drsuapi_changed_objects *m2, + struct drsuapi_getncchanges_state *getnc_state) { return ldb_dn_compare(m2->dn, m1->dn); } @@ -695,8 +696,9 @@ static int site_res_cmp_parent_order(struct drsuapi_changed_objects *m1, /* sort the objects we send first by uSNChanged */ -static int site_res_cmp_dn_usn_order(struct drsuapi_changed_objects *m1, - struct drsuapi_changed_objects *m2) +static int site_res_cmp_usn_order(struct drsuapi_changed_objects *m1, + struct drsuapi_changed_objects *m2, + struct drsuapi_getncchanges_state *getnc_state) { unsigned usnchanged1, usnchanged2; unsigned cn1, cn2; @@ -1789,13 +1791,15 @@ allowed: } if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) { - TYPESAFE_QSORT(changes, - getnc_state->num_records, - site_res_cmp_parent_order); + LDB_TYPESAFE_QSORT(changes, + getnc_state->num_records, + getnc_state, + site_res_cmp_anc_order); } else { - TYPESAFE_QSORT(changes, - getnc_state->num_records, - site_res_cmp_dn_usn_order); + LDB_TYPESAFE_QSORT(changes, + getnc_state->num_records, + getnc_state, + site_res_cmp_usn_order); } for (i=0; i < getnc_state->num_records; i++) { -- 1.7.9.5 From a78414caa811fbf40e3226d9e50bc01938afe15f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sat, 15 Dec 2012 10:18:08 +0100 Subject: [PATCH 18/18] s4:drsuapi: try to behave more like windows for usn order (bug #9508) We don't behave completely like a Windows server, but it's much more identical than before. The partition head is always the first object followed by the rest sorted by uSNChanged. Signed-off-by: Stefan Metzmacher --- source4/rpc_server/drsuapi/getncchanges.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 729b23d..054b32c 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -700,20 +700,27 @@ static int site_res_cmp_usn_order(struct drsuapi_changed_objects *m1, struct drsuapi_changed_objects *m2, struct drsuapi_getncchanges_state *getnc_state) { - unsigned usnchanged1, usnchanged2; - unsigned cn1, cn2; + int ret; - cn1 = ldb_dn_get_comp_num(m1->dn); - cn2 = ldb_dn_get_comp_num(m2->dn); - if (cn1 != cn2) { - return cn1 > cn2 ? 1 : -1; + ret = ldb_dn_compare(getnc_state->ncRoot_dn, m1->dn); + if (ret == 0) { + return -1; } - usnchanged1 = m1->usn; - usnchanged2 = m2->usn; - if (usnchanged1 == usnchanged2) { - return 0; + + ret = ldb_dn_compare(getnc_state->ncRoot_dn, m2->dn); + if (ret == 0) { + return 1; } - return usnchanged1 > usnchanged2 ? 1 : -1; + + if (m1->usn == m2->usn) { + return ldb_dn_compare(m2->dn, m1->dn); + } + + if (m1->usn < m2->usn) { + return -1; + } + + return 1; } -- 1.7.9.5