[PATCH 11/17] Include mimeparse, which is used by subunit/testtools.
Jelmer Vernooij
jelmer at samba.org
Sat Nov 1 13:22:03 MDT 2014
Change-Id: I984c82acc0bc82a165e8ea17d8948c465c786905
Signed-Off-By: Jelmer Vernooij <jelmer at samba.org>
---
auth/credentials/tests/bind.py | 1 +
lib/mimeparse/__init__.py | 0
lib/mimeparse/mimeparse.py | 167 ++++++++++++++++++++++++++
lib/mimeparse/mimeparse_test.py | 68 +++++++++++
lib/mimeparse/setup.py | 51 ++++++++
lib/testtools/Makefile | 4 +-
lib/testtools/NEWS | 2 +
lib/update-external.sh | 5 +
lib/wscript_build | 1 +
selftest/filter-subunit | 1 +
selftest/format-subunit | 1 +
selftest/selftesthelpers.py | 7 +-
source4/dsdb/tests/python/acl.py | 1 +
source4/dsdb/tests/python/deletetest.py | 1 +
source4/dsdb/tests/python/password_lockout.py | 6 +-
source4/dsdb/tests/python/passwords.py | 1 +
source4/dsdb/tests/python/sec_descriptor.py | 1 +
source4/scripting/bin/subunitrun | 1 +
18 files changed, 310 insertions(+), 9 deletions(-)
create mode 100644 lib/mimeparse/__init__.py
create mode 100755 lib/mimeparse/mimeparse.py
create mode 100644 lib/mimeparse/mimeparse_test.py
create mode 100644 lib/mimeparse/setup.py
diff --git a/auth/credentials/tests/bind.py b/auth/credentials/tests/bind.py
index f0788a5..0d6652a 100755
--- a/auth/credentials/tests/bind.py
+++ b/auth/credentials/tests/bind.py
@@ -10,6 +10,7 @@ import time
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
diff --git a/lib/mimeparse/__init__.py b/lib/mimeparse/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/lib/mimeparse/mimeparse.py b/lib/mimeparse/mimeparse.py
new file mode 100755
index 0000000..7fb4e43
--- /dev/null
+++ b/lib/mimeparse/mimeparse.py
@@ -0,0 +1,167 @@
+"""MIME-Type Parser
+
+This module provides basic functions for handling mime-types. It can handle
+matching mime-types against a list of media-ranges. See section 14.1 of the
+HTTP specification [RFC 2616] for a complete explanation.
+
+ http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
+
+Contents:
+ - parse_mime_type(): Parses a mime-type into its component parts.
+ - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q'
+ quality parameter.
+ - quality(): Determines the quality ('q') of a mime-type when
+ compared against a list of media-ranges.
+ - quality_parsed(): Just like quality() except the second parameter must be
+ pre-parsed.
+ - best_match(): Choose the mime-type with the highest quality ('q')
+ from a list of candidates.
+"""
+
+__version__ = '0.1.3'
+__author__ = 'Joe Gregorio'
+__email__ = 'joe at bitworking.org'
+__license__ = 'MIT License'
+__credits__ = ''
+
+
+def parse_mime_type(mime_type):
+ """Parses a mime-type into its component parts.
+
+ Carves up a mime-type and returns a tuple of the (type, subtype, params)
+ where 'params' is a dictionary of all the parameters for the media range.
+ For example, the media range 'application/xhtml;q=0.5' would get parsed
+ into:
+
+ ('application', 'xhtml', {'q', '0.5'})
+ """
+ parts = mime_type.split(';')
+ params = dict([tuple([s.strip() for s in param.split('=', 1)])\
+ for param in parts[1:]
+ ])
+ full_type = parts[0].strip()
+ # Java URLConnection class sends an Accept header that includes a
+ # single '*'. Turn it into a legal wildcard.
+ if full_type == '*':
+ full_type = '*/*'
+ (type, subtype) = full_type.split('/')
+
+ return (type.strip(), subtype.strip(), params)
+
+
+def parse_media_range(range):
+ """Parse a media-range into its component parts.
+
+ Carves up a media range and returns a tuple of the (type, subtype,
+ params) where 'params' is a dictionary of all the parameters for the media
+ range. For example, the media range 'application/*;q=0.5' would get parsed
+ into:
+
+ ('application', '*', {'q', '0.5'})
+
+ In addition this function also guarantees that there is a value for 'q'
+ in the params dictionary, filling it in with a proper default if
+ necessary.
+ """
+ (type, subtype, params) = parse_mime_type(range)
+ if 'q' not in params or not params['q'] or \
+ not float(params['q']) or float(params['q']) > 1\
+ or float(params['q']) < 0:
+ params['q'] = '1'
+
+ return (type, subtype, params)
+
+
+def fitness_and_quality_parsed(mime_type, parsed_ranges):
+ """Find the best match for a mime-type amongst parsed media-ranges.
+
+ Find the best match for a given mime-type against a list of media_ranges
+ that have already been parsed by parse_media_range(). Returns a tuple of
+ the fitness value and the value of the 'q' quality parameter of the best
+ match, or (-1, 0) if no match was found. Just as for quality_parsed(),
+ 'parsed_ranges' must be a list of parsed media ranges.
+ """
+ best_fitness = -1
+ best_fit_q = 0
+ (target_type, target_subtype, target_params) =\
+ parse_media_range(mime_type)
+ for (type, subtype, params) in parsed_ranges:
+ type_match = (type == target_type or\
+ type == '*' or\
+ target_type == '*')
+ subtype_match = (subtype == target_subtype or\
+ subtype == '*' or\
+ target_subtype == '*')
+ if type_match and subtype_match:
+ param_matches = sum([1 for (key, value) in \
+ target_params.items() if key != 'q' and \
+ key in params and value == params[key]], 0)
+ fitness = (type == target_type) and 100 or 0
+ fitness += (subtype == target_subtype) and 10 or 0
+ fitness += param_matches
+ if fitness > best_fitness:
+ best_fitness = fitness
+ best_fit_q = params['q']
+
+ return best_fitness, float(best_fit_q)
+
+
+def quality_parsed(mime_type, parsed_ranges):
+ """Find the best match for a mime-type amongst parsed media-ranges.
+
+ Find the best match for a given mime-type against a list of media_ranges
+ that have already been parsed by parse_media_range(). Returns the 'q'
+ quality parameter of the best match, 0 if no match was found. This function
+ bahaves the same as quality() except that 'parsed_ranges' must be a list of
+ parsed media ranges. """
+
+ return fitness_and_quality_parsed(mime_type, parsed_ranges)[1]
+
+
+def quality(mime_type, ranges):
+ """Return the quality ('q') of a mime-type against a list of media-ranges.
+
+ Returns the quality 'q' of a mime-type when compared against the
+ media-ranges in ranges. For example:
+
+ >>> quality('text/html','text/*;q=0.3, text/html;q=0.7,
+ text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5')
+ 0.7
+
+ """
+ parsed_ranges = [parse_media_range(r) for r in ranges.split(',')]
+
+ return quality_parsed(mime_type, parsed_ranges)
+
+
+def best_match(supported, header):
+ """Return mime-type with the highest quality ('q') from list of candidates.
+
+ Takes a list of supported mime-types and finds the best match for all the
+ media-ranges listed in header. The value of header must be a string that
+ conforms to the format of the HTTP Accept: header. The value of 'supported'
+ is a list of mime-types. The list of supported mime-types should be sorted
+ in order of increasing desirability, in case of a situation where there is
+ a tie.
+
+ >>> best_match(['application/xbel+xml', 'text/xml'],
+ 'text/*;q=0.5,*/*; q=0.1')
+ 'text/xml'
+ """
+ split_header = _filter_blank(header.split(','))
+ parsed_header = [parse_media_range(r) for r in split_header]
+ weighted_matches = []
+ pos = 0
+ for mime_type in supported:
+ weighted_matches.append((fitness_and_quality_parsed(mime_type,
+ parsed_header), pos, mime_type))
+ pos += 1
+ weighted_matches.sort()
+
+ return weighted_matches[-1][0][1] and weighted_matches[-1][2] or ''
+
+
+def _filter_blank(i):
+ for s in i:
+ if s.strip():
+ yield s
diff --git a/lib/mimeparse/mimeparse_test.py b/lib/mimeparse/mimeparse_test.py
new file mode 100644
index 0000000..969cbf3
--- /dev/null
+++ b/lib/mimeparse/mimeparse_test.py
@@ -0,0 +1,68 @@
+"""
+Python tests for Mime-Type Parser.
+
+This module loads a json file and converts the tests specified therein to a set
+of PyUnitTestCases. Then it uses PyUnit to run them and report their status.
+"""
+__version__ = "0.1"
+__author__ = 'Ade Oshineye'
+__email__ = "ade at oshineye.com"
+__credits__ = ""
+
+import mimeparse
+import unittest
+from functools import partial
+# Conditional import to support Python 2.5
+try:
+ import json
+except ImportError:
+ import simplejson as json
+
+def test_parse_media_range(args, expected):
+ expected = tuple(expected)
+ result = mimeparse.parse_media_range(args)
+ message = "Expected: '%s' but got %s" % (expected, result)
+ assert expected == result, message
+
+def test_quality(args, expected):
+ result = mimeparse.quality(args[0], args[1])
+ message = "Expected: '%s' but got %s" % (expected, result)
+ assert expected == result, message
+
+def test_best_match(args, expected):
+ result = mimeparse.best_match(args[0], args[1])
+ message = "Expected: '%s' but got %s" % (expected, result)
+ assert expected == result, message
+
+def test_parse_mime_type(args, expected):
+ expected = tuple(expected)
+ result = mimeparse.parse_mime_type(args)
+ message = "Expected: '%s' but got %s" % (expected, result)
+ assert expected == result, message
+
+def add_tests(suite, json_object, func_name, test_func):
+ test_data = json_object[func_name]
+ for test_datum in test_data:
+ args, expected = test_datum[0], test_datum[1]
+ desc = "%s(%s) with expected result: %s" % (func_name, str(args), str(expected))
+ if len(test_datum) == 3:
+ desc = test_datum[2] + " : " + desc
+ func = partial(test_func, *(args, expected))
+ func.__name__ = test_func.__name__
+ testcase = unittest.FunctionTestCase(func, description=desc)
+ suite.addTest(testcase)
+
+def run_tests():
+ json_object = json.load(open("testdata.json"))
+
+ suite = unittest.TestSuite()
+ add_tests(suite, json_object, "parse_media_range", test_parse_media_range)
+ add_tests(suite, json_object, "quality", test_quality)
+ add_tests(suite, json_object, "best_match", test_best_match)
+ add_tests(suite, json_object, "parse_mime_type", test_parse_mime_type)
+
+ test_runner = unittest.TextTestRunner(verbosity=1)
+ test_runner.run(suite)
+
+if __name__ == "__main__":
+ run_tests()
diff --git a/lib/mimeparse/setup.py b/lib/mimeparse/setup.py
new file mode 100644
index 0000000..e559f9d
--- /dev/null
+++ b/lib/mimeparse/setup.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+#old way
+from distutils.core import setup
+
+#new way
+#from setuptools import setup, find_packages
+
+setup(name='mimeparse',
+ version='0.1.4',
+ description='A module provides basic functions for parsing mime-type names and matching them against a list of media-ranges.',
+ long_description="""
+This module provides basic functions for handling mime-types. It can handle
+matching mime-types against a list of media-ranges. See section 14.1 of
+the HTTP specification [RFC 2616] for a complete explanation.
+
+ http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
+
+Contents:
+ - parse_mime_type(): Parses a mime-type into its component parts.
+ - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' quality parameter.
+ - quality(): Determines the quality ('q') of a mime-type when compared against a list of media-ranges.
+ - quality_parsed(): Just like quality() except the second parameter must be pre-parsed.
+ - best_match(): Choose the mime-type with the highest quality ('q') from a list of candidates.
+ """,
+ classifiers=[
+ # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: MIT License',
+ 'Programming Language :: Python',
+ 'Topic :: Internet :: WWW/HTTP',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.5',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.1',
+ 'Programming Language :: Python :: 3.2',
+ 'Programming Language :: Python :: 3.3',
+ ],
+ keywords='mime-type',
+ author='Joe Gregorio',
+ author_email='joe at bitworking.org',
+ maintainer='Joe Gregorio',
+ maintainer_email='joe at bitworking.org',
+ url='http://code.google.com/p/mimeparse/',
+ license='MIT',
+ py_modules=['mimeparse']
+ )
+
diff --git a/lib/testtools/Makefile b/lib/testtools/Makefile
index ccaa776..c637123 100644
--- a/lib/testtools/Makefile
+++ b/lib/testtools/Makefile
@@ -21,11 +21,11 @@ prerelease:
-rm MANIFEST
release:
- ./setup.py sdist upload --sign
+ ./setup.py sdist bdist_wheel upload --sign
$(PYTHON) scripts/_lp_release.py
snapshot: prerelease
- ./setup.py sdist
+ ./setup.py sdist bdist_wheel
### Documentation ###
diff --git a/lib/testtools/NEWS b/lib/testtools/NEWS
index 640a952..647ceff 100644
--- a/lib/testtools/NEWS
+++ b/lib/testtools/NEWS
@@ -16,6 +16,8 @@ Changes
* Make `testtools.content.text_content` error if anything other than text
is given as content. (Thomi Richards)
+* We now publish wheels of testtools. (Robert Collins, #issue84)
+
1.1.0
~~~~~
diff --git a/lib/update-external.sh b/lib/update-external.sh
index 1d30721..779e7f5 100755
--- a/lib/update-external.sh
+++ b/lib/update-external.sh
@@ -41,4 +41,9 @@ git clone git://github.com/testing-cabal/extras "$WORKDIR/extras"
rm -rf "$WORKDIR/extras/.git"
rsync -avz --delete "$WORKDIR/extras/" "$LIBDIR/extras/"
+echo "Updating mimeparse..."
+svn co http://mimeparse.googlecode.com/svn/trunk/ "$WORKDIR/mimeparse"
+rm -rf "$WORKDIR/mimeparse/.svn"
+rsync -avz --delete "$WORKDIR/mimeparse/" "$LIBDIR/mimeparse/"
+
rm -rf "$WORKDIR"
diff --git a/lib/wscript_build b/lib/wscript_build
index c6ca778..8118a6c 100644
--- a/lib/wscript_build
+++ b/lib/wscript_build
@@ -8,6 +8,7 @@ external_libs = {
"subunit": "subunit/python/subunit",
"testtools": "testtools/testtools",
"extras": "extras/extras",
+ "mimeparse": "mimeparse",
}
list = []
diff --git a/selftest/filter-subunit b/selftest/filter-subunit
index 654ad9b..2ce9584 100755
--- a/selftest/filter-subunit
+++ b/selftest/filter-subunit
@@ -25,6 +25,7 @@ import signal
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/mimeparse"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/extras"))
import subunithelper
diff --git a/selftest/format-subunit b/selftest/format-subunit
index 1b33547..5da56be 100755
--- a/selftest/format-subunit
+++ b/selftest/format-subunit
@@ -11,6 +11,7 @@ import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/mimeparse"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/extras"))
import subunithelper
diff --git a/selftest/selftesthelpers.py b/selftest/selftesthelpers.py
index 4724e8a..f2fbfeb 100644
--- a/selftest/selftesthelpers.py
+++ b/selftest/selftesthelpers.py
@@ -68,8 +68,8 @@ else:
python = os.getenv("PYTHON", "python")
# Set a default value, overridden if we find a working one on the system
-tap2subunit = "PYTHONPATH=%s/lib/subunit/python:%s/lib/testtools:%s/lib/extras %s %s/lib/subunit/filters/tap2subunit" % (srcdir(), srcdir(), srcdir(), python, srcdir())
-subunit2to1 = "PYTHONPATH=%s/lib/subunit/python:%s/lib/testtools:%s/lib/extras %s %s/lib/subunit/filters/subunit-2to1" % (srcdir(), srcdir(), srcdir(), python, srcdir())
+tap2subunit = "PYTHONPATH=%s/lib/subunit/python:%s/lib/testtools:%s/lib/extras:%s/lib/mimeparse %s %s/lib/subunit/filters/tap2subunit" % (srcdir(), srcdir(), srcdir(), srcdir(), python, srcdir())
+subunit2to1 = "PYTHONPATH=%s/lib/subunit/python:%s/lib/testtools:%s/lib/extras:%s/lib/mimeparse %s %s/lib/subunit/filters/subunit-2to1" % (srcdir(), srcdir(), srcdir(), srcdir(), python, srcdir())
sub = subprocess.Popen("tap2subunit", stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
@@ -174,7 +174,8 @@ def planpythontestsuite(env, module, name=None, extra_path=[]):
pypath.extend([
"%s/lib/subunit/python" % srcdir(),
"%s/lib/testtools" % srcdir(),
- "%s/lib/extras" % srcdir()])
+ "%s/lib/extras" % srcdir(),
+ "%s/lib/mimeparse" % srcdir()])
args = [python, "-m", "subunit.run", "$LISTOPT", "$LOADLIST", module]
if pypath:
args.insert(0, "PYTHONPATH=%s" % ":".join(["$PYTHONPATH"] + pypath))
diff --git a/source4/dsdb/tests/python/acl.py b/source4/dsdb/tests/python/acl.py
index 5c2a257..9ef97ee 100755
--- a/source4/dsdb/tests/python/acl.py
+++ b/source4/dsdb/tests/python/acl.py
@@ -8,6 +8,7 @@ import base64
import re
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
diff --git a/source4/dsdb/tests/python/deletetest.py b/source4/dsdb/tests/python/deletetest.py
index 43bb158..48c81f8 100755
--- a/source4/dsdb/tests/python/deletetest.py
+++ b/source4/dsdb/tests/python/deletetest.py
@@ -7,6 +7,7 @@ import os
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
diff --git a/source4/dsdb/tests/python/password_lockout.py b/source4/dsdb/tests/python/password_lockout.py
index b77209a..fa13edf 100755
--- a/source4/dsdb/tests/python/password_lockout.py
+++ b/source4/dsdb/tests/python/password_lockout.py
@@ -14,6 +14,7 @@ import time
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
@@ -23,13 +24,10 @@ import samba.getopt as options
from samba.auth import system_session
from samba.credentials import Credentials, DONT_USE_KERBEROS, MUST_USE_KERBEROS
from ldb import SCOPE_BASE, LdbError
-from ldb import ERR_ATTRIBUTE_OR_VALUE_EXISTS
-from ldb import ERR_UNWILLING_TO_PERFORM, ERR_INSUFFICIENT_ACCESS_RIGHTS
-from ldb import ERR_NO_SUCH_ATTRIBUTE
from ldb import ERR_CONSTRAINT_VIOLATION
from ldb import ERR_INVALID_CREDENTIALS
from ldb import Message, MessageElement, Dn
-from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
+from ldb import FLAG_MOD_REPLACE
from samba import gensec, dsdb
from samba.samdb import SamDB
import samba.tests
diff --git a/source4/dsdb/tests/python/passwords.py b/source4/dsdb/tests/python/passwords.py
index 83e29fa..b67347e 100755
--- a/source4/dsdb/tests/python/passwords.py
+++ b/source4/dsdb/tests/python/passwords.py
@@ -16,6 +16,7 @@ import os
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
diff --git a/source4/dsdb/tests/python/sec_descriptor.py b/source4/dsdb/tests/python/sec_descriptor.py
index 5b9816b..54774a4 100755
--- a/source4/dsdb/tests/python/sec_descriptor.py
+++ b/source4/dsdb/tests/python/sec_descriptor.py
@@ -10,6 +10,7 @@ import random
sys.path.insert(0, "bin/python")
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
diff --git a/source4/scripting/bin/subunitrun b/source4/scripting/bin/subunitrun
index 6563c88..91dc158 100755
--- a/source4/scripting/bin/subunitrun
+++ b/source4/scripting/bin/subunitrun
@@ -39,6 +39,7 @@ sys.path.insert(0, "bin/python")
import optparse
import samba
+samba.ensure_external_module("mimeparse", "mimeparse")
samba.ensure_external_module("extras", "extras")
samba.ensure_external_module("testtools", "testtools")
samba.ensure_external_module("subunit", "subunit/python")
--
2.1.1
More information about the samba-technical
mailing list