From eee7242e077c4894d4489a91ed8de662c5372110 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 8 Feb 2018 00:47:38 -0800 Subject: [PATCH 1/4] Preserve mapping order in ChainMap() --- Lib/collections/__init__.py | 8 +++++++- Lib/test/test_collections.py | 5 +++++ .../next/Library/2018-02-08-00-47-07.bpo-32792.NtyDb4.rst | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2018-02-08-00-47-07.bpo-32792.NtyDb4.rst diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index f0b41fd9d10d3e..3a614a093a7ae4 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -920,7 +920,13 @@ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible def __iter__(self): - return iter(set().union(*self.maps)) + seen = set() + seen_add = seen.add + for mapping in self.maps: + for k in mapping: + if k not in seen: + seen_add(k) + yield k def __contains__(self, key): return any(key in m for m in self.maps) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index a55239e57302e7..3e2de86a8edb68 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -141,6 +141,11 @@ def __missing__(self, key): with self.assertRaises(KeyError): d.popitem() + def test_order_preservation(self): + d = ChainMap(dict(a=1, b=2), dict(b=3, c=4), dict(c=5, d=6), + dict(d=7, e=8), dict(e=9, f=10, g=11)) + self.assertEqual(list(d), ['a', 'b', 'c', 'd', 'e', 'f', 'g']) + def test_dict_coercion(self): d = ChainMap(dict(a=1, b=2), dict(b=20, c=30)) self.assertEqual(dict(d), dict(a=1, b=2, c=30)) diff --git a/Misc/NEWS.d/next/Library/2018-02-08-00-47-07.bpo-32792.NtyDb4.rst b/Misc/NEWS.d/next/Library/2018-02-08-00-47-07.bpo-32792.NtyDb4.rst new file mode 100644 index 00000000000000..1f7df62cc3e12e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-08-00-47-07.bpo-32792.NtyDb4.rst @@ -0,0 +1 @@ +collections.ChainMap() preserves the order of the underlying mappings. From 6ce6c9929474e635a336d613b824c14f04401248 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 8 Feb 2018 00:58:44 -0800 Subject: [PATCH 2/4] Use OrderedDict for the test data --- Lib/test/test_collections.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 3e2de86a8edb68..bfcdddc31fa30e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -142,9 +142,10 @@ def __missing__(self, key): d.popitem() def test_order_preservation(self): - d = ChainMap(dict(a=1, b=2), dict(b=3, c=4), dict(c=5, d=6), - dict(d=7, e=8), dict(e=9, f=10, g=11)) - self.assertEqual(list(d), ['a', 'b', 'c', 'd', 'e', 'f', 'g']) + OD = OrderedDict + d = ChainMap(OD(a=1, b=2), OD(b=3, c=4), OD(c=5, d=6), + OD(d=7, e=8), OD(e=9, f=10, g=11, h=12)) + self.assertEqual(list(d), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']) def test_dict_coercion(self): d = ChainMap(dict(a=1, b=2), dict(b=20, c=30)) From 3f9425ca5b6f3dba22840894c0026d06595bfe61 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 8 Feb 2018 01:03:30 -0800 Subject: [PATCH 3/4] All test items() which depends on __iter__(). --- Lib/test/test_collections.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index bfcdddc31fa30e..a60a0ba155e061 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -146,6 +146,9 @@ def test_order_preservation(self): d = ChainMap(OD(a=1, b=2), OD(b=3, c=4), OD(c=5, d=6), OD(d=7, e=8), OD(e=9, f=10, g=11, h=12)) self.assertEqual(list(d), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']) + self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 4), + ('d', 6), ('e', 8), ('f', 10), + ('g', 11), ('h', 12)]) def test_dict_coercion(self): d = ChainMap(dict(a=1, b=2), dict(b=20, c=30)) From 82fa93d40406428eeddc2dc0a87fa87630950d2e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 10 Feb 2018 14:39:03 -0800 Subject: [PATCH 4/4] Adopt Serhiy's suggested alternative algorithm which is faster and gives a more useful ordering. --- Lib/collections/__init__.py | 11 ++++------- Lib/test/test_collections.py | 22 +++++++++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 3a614a093a7ae4..9a753db71caeae 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -920,13 +920,10 @@ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible def __iter__(self): - seen = set() - seen_add = seen.add - for mapping in self.maps: - for k in mapping: - if k not in seen: - seen_add(k) - yield k + d = {} + for mapping in reversed(self.maps): + d.update(mapping) # reuses stored hash values if possible + return iter(d) def __contains__(self, key): return any(key in m for m in self.maps) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index a60a0ba155e061..2099d236d0c4a1 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -142,13 +142,21 @@ def __missing__(self, key): d.popitem() def test_order_preservation(self): - OD = OrderedDict - d = ChainMap(OD(a=1, b=2), OD(b=3, c=4), OD(c=5, d=6), - OD(d=7, e=8), OD(e=9, f=10, g=11, h=12)) - self.assertEqual(list(d), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']) - self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 4), - ('d', 6), ('e', 8), ('f', 10), - ('g', 11), ('h', 12)]) + d = ChainMap( + OrderedDict(j=0, h=88888), + OrderedDict(), + OrderedDict(i=9999, d=4444, c=3333), + OrderedDict(f=666, b=222, g=777, c=333, h=888), + OrderedDict(), + OrderedDict(e=55, b=22), + OrderedDict(a=1, b=2, c=3, d=4, e=5), + OrderedDict(), + ) + self.assertEqual(''.join(d), 'abcdefghij') + self.assertEqual(list(d.items()), + [('a', 1), ('b', 222), ('c', 3333), ('d', 4444), + ('e', 55), ('f', 666), ('g', 777), ('h', 88888), + ('i', 9999), ('j', 0)]) def test_dict_coercion(self): d = ChainMap(dict(a=1, b=2), dict(b=20, c=30))