[SCM] Samba Shared Repository - branch master updated
Ralph Böhme
slow at samba.org
Mon Jul 24 16:16:02 UTC 2023
The branch, master has been updated
via 61c951e063e mdscli: correct handling of in-progress searches
via 424af98c894 mdscli: increase MAX_SLQ_COUNT
via b8e0f02f081 mdscli: increase MAX_SLQ_TOCIDX
via 1149d497b35 mdssvc: increase MAX_SLQ_TOC
via 68bb582bc51 mdssvc: introduce MAX_MDSCMD_SIZE
via c2b4fe3fb7c mdscli: add fragmentation support
via 27980c87c9b mdssvc: remove duplicate define of MAX_SL_FRAGMENT_SIZE
via 566427c4f0e librpc/idl: mdssvc: unkn4 field is a fragment indicator
from 5442c47dad2 libsmb: increase a debug level when site-aware DC lookup failed
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 61c951e063ecf98c6b61e03885eaa6d4a48f763b
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 15:12:49 2023 +0200
mdscli: correct handling of in-progress searches
If a query is still being processed on the server and there no results yet,
macOS returns 0x23.
For now just implements this as dumb polling once a second in mdsearch and the
Python bindings.
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
Autobuild-User(master): Ralph Böhme <slow at samba.org>
Autobuild-Date(master): Mon Jul 24 16:15:16 UTC 2023 on atb-devel-224
commit 424af98c894b20b36fb7f7081caab1b8634c7633
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 16:54:13 2023 +0200
mdscli: increase MAX_SLQ_COUNT
Now that the client supports fragementation, when procesing reassambled large
result sets from macOS Spotlight server, it's possible to hit this limit in the
client. Let's just increase it to some larger value.
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit b8e0f02f081a3d17f2147b6deeeeb7b722a7704b
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 16:52:19 2023 +0200
mdscli: increase MAX_SLQ_TOCIDX
Now that the client supports fragementation, when procesing reassambled large
result sets from macOS Spotlight server, it's possible to hit this limit in the
client. Let's just increase it to some larger value.
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit 1149d497b35faf375b9011241af821d73b90a337
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 16:50:58 2023 +0200
mdssvc: increase MAX_SLQ_TOC
Now that the client supports fragementation, when procesing reassambled large
result sets from macOS Spotlight server, it's possible to hit this limit in the
client. Let's just increase it to some larger value.
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit 68bb582bc513448a9b16705ca566539cfaf45882
Author: Ralph Boehme <slow at samba.org>
Date: Mon Jul 17 14:51:54 2023 +0200
mdssvc: introduce MAX_MDSCMD_SIZE
Allow larger value for the reassembled mdscmd's. Now that the client supports
fragementation, when processing reassambled large result sets from macOS
Spotlight server, it's possible to hit this limit in the unmarshalling
code. Let's just increase it to some larger value.
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit c2b4fe3fb7c71579541829aca7c112bc2c0eebeb
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 10:45:52 2023 +0200
mdscli: add fragmentation support
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit 27980c87c9b77d6d8a4de6210e13fa20ff870ba9
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 14:59:12 2023 +0200
mdssvc: remove duplicate define of MAX_SL_FRAGMENT_SIZE
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
commit 566427c4f0e8c0d9a6ceae4807f6fe6c2fceea1c
Author: Ralph Boehme <slow at samba.org>
Date: Thu Apr 20 10:44:58 2023 +0200
librpc/idl: mdssvc: unkn4 field is a fragment indicator
Signed-off-by: Ralph Boehme <slow at samba.org>
Reviewed-by: Noel Power <noel.power at suse.com>
-----------------------------------------------------------------------
Summary of changes:
librpc/idl/mdssvc.idl | 2 +-
source3/rpc_client/cli_mdssvc.c | 76 +++++++++++++++++++++++++++++----
source3/rpc_client/py_mdscli.c | 7 ++-
source3/rpc_server/mdssvc/marshalling.c | 14 +++---
source3/rpc_server/mdssvc/marshalling.h | 1 +
source3/rpc_server/mdssvc/mdssvc.h | 1 -
source3/utils/mdsearch.c | 10 ++++-
7 files changed, 91 insertions(+), 20 deletions(-)
Changeset truncated at 500 lines:
diff --git a/librpc/idl/mdssvc.idl b/librpc/idl/mdssvc.idl
index e7004f07419..b77474713c9 100644
--- a/librpc/idl/mdssvc.idl
+++ b/librpc/idl/mdssvc.idl
@@ -41,7 +41,7 @@ interface mdssvc
[in] uint32 unkn1, /* always 0, status ? */
[in] uint32 device_id,
[in] uint32 unkn3, /* = mdssvc_open.unkn2 ? */
- [in] uint32 unkn4, /* always 0 ? */
+ [in] uint32 next_fragment, /* Set to 1 to request next fragment*/
[in] uint32 flags, /* always 0x6b000001 ? */
[in] mdssvc_blob request_blob,
[in] uint32 unkn5, /* always 0 ? */
diff --git a/source3/rpc_client/cli_mdssvc.c b/source3/rpc_client/cli_mdssvc.c
index 753bc2e52ed..839cd62ce86 100644
--- a/source3/rpc_client/cli_mdssvc.c
+++ b/source3/rpc_client/cli_mdssvc.c
@@ -532,10 +532,13 @@ fail:
}
struct mdscli_get_results_state {
+ struct tevent_context *ev;
struct mdscli_search_ctx *search;
struct mdssvc_blob request_blob;
- struct mdssvc_blob response_blob;
+ struct mdssvc_blob response_fragment;
+ DATA_BLOB response_data;
uint64_t *cnids;
+ uint32_t fragment;
};
static void mdscli_get_results_cmd_done(struct tevent_req *subreq);
@@ -557,6 +560,7 @@ struct tevent_req *mdscli_get_results_send(
}
*state = (struct mdscli_get_results_state) {
+ .ev = ev,
.search = search,
};
@@ -583,8 +587,8 @@ struct tevent_req *mdscli_get_results_send(
mdscli_ctx->max_fragment_size,
0,
0,
- &mdscli_ctx->mdscmd_cmd.fragment,
- &state->response_blob,
+ &state->fragment,
+ &state->response_fragment,
&mdscli_ctx->mdscmd_cmd.unkn9);
if (tevent_req_nomem(subreq, req)) {
return tevent_req_post(req, ev);
@@ -600,8 +604,11 @@ static void mdscli_get_results_cmd_done(struct tevent_req *subreq)
subreq, struct tevent_req);
struct mdscli_get_results_state *state = tevent_req_data(
req, struct mdscli_get_results_state);
+ struct mdscli_ctx *mdscli_ctx = state->search->mdscli_ctx;
+ size_t oldsize, newsize;
DALLOC_CTX *d = NULL;
uint64_t *uint64p = NULL;
+ bool search_in_progress = false;
sl_cnids_t *cnids = NULL;
size_t ncnids;
size_t i;
@@ -615,14 +622,66 @@ static void mdscli_get_results_cmd_done(struct tevent_req *subreq)
return;
}
+ oldsize = state->response_data.length;
+ newsize = oldsize + state->response_fragment.length;
+ if (newsize < oldsize) {
+ tevent_req_nterror(req, NT_STATUS_INTEGER_OVERFLOW);
+ return;
+ }
+
+ ok = data_blob_realloc(state, &state->response_data, newsize);
+ if (!ok) {
+ tevent_req_nterror(req, NT_STATUS_NO_MEMORY);
+ return;
+ }
+ (void)memcpy(state->response_data.data + oldsize,
+ state->response_fragment.spotlight_blob,
+ state->response_fragment.length);
+
+ TALLOC_FREE(state->response_fragment.spotlight_blob);
+ state->response_fragment.length = 0;
+ state->response_fragment.size = 0;
+
+ if (state->fragment != 0) {
+ subreq = dcerpc_mdssvc_cmd_send(
+ state,
+ state->ev,
+ mdscli_ctx->bh,
+ &mdscli_ctx->ph,
+ 0,
+ mdscli_ctx->dev,
+ mdscli_ctx->mdscmd_open.unkn2,
+ 1,
+ mdscli_ctx->flags,
+ state->request_blob,
+ 0,
+ mdscli_ctx->max_fragment_size,
+ 1,
+ mdscli_ctx->max_fragment_size,
+ 0,
+ 0,
+ &state->fragment,
+ &state->response_fragment,
+ &mdscli_ctx->mdscmd_cmd.unkn9);
+ if (tevent_req_nomem(subreq, req)) {
+ tevent_req_post(req, state->ev);
+ return;
+ }
+ tevent_req_set_callback(subreq,
+ mdscli_get_results_cmd_done,
+ req);
+ mdscli_ctx->async_pending++;
+ return;
+ }
+
d = dalloc_new(state);
if (tevent_req_nomem(d, req)) {
return;
}
ok = sl_unpack(d,
- (char *)state->response_blob.spotlight_blob,
- state->response_blob.length);
+ (char *)state->response_data.data,
+ state->response_data.length);
if (!ok) {
tevent_req_nterror(req, NT_STATUS_INTERNAL_ERROR);
return;
@@ -638,9 +697,8 @@ static void mdscli_get_results_cmd_done(struct tevent_req *subreq)
}
if (*uint64p == 35) {
- DBG_DEBUG("search done: %s", dalloc_dump(d, 0));
- tevent_req_done(req);
- return;
+ DBG_DEBUG("Search in progress\n");
+ search_in_progress = true;
}
cnids = dalloc_get(d, "DALLOC_CTX", 0, "sl_cnids_t", 1);
@@ -651,7 +709,7 @@ static void mdscli_get_results_cmd_done(struct tevent_req *subreq)
}
ncnids = dalloc_size(cnids->ca_cnids);
- if (ncnids == 0) {
+ if (ncnids == 0 && !search_in_progress) {
tevent_req_nterror(req, NT_STATUS_NO_MORE_MATCHES);
return;
}
diff --git a/source3/rpc_client/py_mdscli.c b/source3/rpc_client/py_mdscli.c
index 290be3f86d9..2594200ecfe 100644
--- a/source3/rpc_client/py_mdscli.c
+++ b/source3/rpc_client/py_mdscli.c
@@ -88,6 +88,7 @@ static PyObject *search_get_results(PyObject *self,
* py_dcerpc_interface_init_helper()
* -> dcerpc_pipe_connect()
*/
+again:
req = mdscli_get_results_send(frame,
pipe->ev,
search);
@@ -102,13 +103,17 @@ static PyObject *search_get_results(PyObject *self,
}
status = mdscli_get_results_recv(req, frame, &cnids);
+ TALLOC_FREE(req);
+ if (NT_STATUS_EQUAL(status, NT_STATUS_PENDING)) {
+ sleep(1);
+ goto again;
+ }
if (!NT_STATUS_IS_OK(status) &&
!NT_STATUS_EQUAL(status, NT_STATUS_NO_MORE_MATCHES))
{
PyErr_SetNTSTATUS(status);
goto out;
}
- TALLOC_FREE(req);
result = Py_BuildValue("[]");
diff --git a/source3/rpc_server/mdssvc/marshalling.c b/source3/rpc_server/mdssvc/marshalling.c
index 5f866d7fb6e..3bbd5889aaa 100644
--- a/source3/rpc_server/mdssvc/marshalling.c
+++ b/source3/rpc_server/mdssvc/marshalling.c
@@ -34,9 +34,9 @@
* The total buffersize for S-RPC packets is typically limited to 64k,
* so we can only store so many elements there anyway.
*/
-#define MAX_SLQ_TOC 1024*8
-#define MAX_SLQ_TOCIDX 1024
-#define MAX_SLQ_COUNT 4096
+#define MAX_SLQ_TOC 1024*64
+#define MAX_SLQ_TOCIDX 1024*8
+#define MAX_SLQ_COUNT 1024*64
#define MAX_SL_STRLEN 1024
/******************************************************************************
@@ -667,12 +667,12 @@ static ssize_t sl_unpack_tag(const char *buf,
tag->count = val >> 32;
tag->length = tag->count * 8;
- if (tag->size > MAX_SL_FRAGMENT_SIZE) {
+ if (tag->size > MAX_MDSCMD_SIZE) {
DEBUG(1,("%s: size limit %zu\n", __func__, tag->size));
return -1;
}
- if (tag->length > MAX_SL_FRAGMENT_SIZE) {
+ if (tag->length > MAX_MDSCMD_SIZE) {
DEBUG(1,("%s: length limit %zu\n", __func__, tag->length));
return -1;
}
@@ -963,7 +963,7 @@ static ssize_t sl_unpack_cpx(DALLOC_CTX *query,
return -1;
}
slen = tag.size - 16 + tag.count;
- if (slen > MAX_SL_FRAGMENT_SIZE) {
+ if (slen > MAX_MDSCMD_SIZE) {
return -1;
}
@@ -1312,7 +1312,7 @@ bool sl_unpack(DALLOC_CTX *query, const char *buf, size_t bufsize)
uint64_t toc_offset;
struct sl_tag toc_tag;
- if (bufsize > MAX_SL_FRAGMENT_SIZE) {
+ if (bufsize > MAX_MDSCMD_SIZE) {
return false;
}
diff --git a/source3/rpc_server/mdssvc/marshalling.h b/source3/rpc_server/mdssvc/marshalling.h
index 2cc1b44712c..ccc8b442268 100644
--- a/source3/rpc_server/mdssvc/marshalling.h
+++ b/source3/rpc_server/mdssvc/marshalling.h
@@ -27,6 +27,7 @@
#include "librpc/gen_ndr/mdssvc.h"
#define MAX_SL_FRAGMENT_SIZE 0xFFFFF
+#define MAX_MDSCMD_SIZE 0xFFFFFF
/* Can be ored and used as flags */
#define SL_ENC_LITTLE_ENDIAN 1
diff --git a/source3/rpc_server/mdssvc/mdssvc.h b/source3/rpc_server/mdssvc/mdssvc.h
index 38029232242..5836caff740 100644
--- a/source3/rpc_server/mdssvc/mdssvc.h
+++ b/source3/rpc_server/mdssvc/mdssvc.h
@@ -34,7 +34,6 @@
#undef TRUE
#undef FALSE
-#define MAX_SL_FRAGMENT_SIZE 0xFFFFF
#define MAX_SL_RESULTS 100
#define MAX_SL_RUNTIME 30
#define MDS_TRACKER_ASYNC_TIMEOUT_MS 250
diff --git a/source3/utils/mdsearch.c b/source3/utils/mdsearch.c
index eddb83874cb..1472b5cfa32 100644
--- a/source3/utils/mdsearch.c
+++ b/source3/utils/mdsearch.c
@@ -197,12 +197,20 @@ int main(int argc, char **argv)
}
break;
}
+
+ ncnids = talloc_array_length(cnids);
+
+ if (NT_STATUS_EQUAL(status, NT_STATUS_PENDING) &&
+ ncnids == 0)
+ {
+ sleep(1);
+ continue;
+ }
if (!NT_STATUS_IS_OK(status)) {
printf("mdscli_get_results failed\n");
goto fail_free_messaging;
}
- ncnids = talloc_array_length(cnids);
if (ncnids == 0) {
break;
}
--
Samba Shared Repository
More information about the samba-cvs
mailing list