From bb7ab509eb2e7aa38f6f9a114322db17d5c199dd Mon Sep 17 00:00:00 2001 From: Ramesh RV Date: Sun, 23 Sep 2018 17:46:50 +0530 Subject: [PATCH 1/2] Addition of new feature (get_all_keys) --- README.rst | 39 ++++++++- nested_lookup/__init__.py | 2 +- nested_lookup/nested_lookup.py | 44 +++++++++- setup.py | 2 +- test_nested_loopkup.py | 141 +++++++++++++++++++++++++++------ 5 files changed, 198 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 92cb08f..374a2ab 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,9 @@ nested_lookup .. image:: https://img.shields.io/badge/pypi-0.1.5-green.svg :target: https://pypi.python.org/pypi/nested-lookup -A small Python library which enables key lookups on deeply nested documents. +A small Python library which enables: +1. key lookups on deeply nested documents. +2. fetching all keys from a nested dictionary. Documents may be built out of dictionaries (dicts) and/or lists. @@ -42,6 +44,11 @@ quick tutorial >>> print(nested_lookup('taco', document)) [42, 69] + >>> from nested_lookup import get_all_keys + + >>> get_all_keys(document) + ['taco', 'salsa', 'burrito', 'taco'] + longer tutorial =============== @@ -109,6 +116,36 @@ Additionally, if you also needed the matched key names, you could do this: } +Tutorial to get all keys from a nested dictionary + +.. code-block:: python + + sample_data = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": { + "processor_name": "Intel Core i7", + "processor_speed": "2.7 GHz", + "core_details": { + "total_numberof_cores": "4", + "l2_cache(per_core)": "256 KB" + } + }, + "total_number_of_cores": "4", + "memory": "16 GB", + }, + "os_details": { + "product_version": "10.13.6", + "build_version": "17G65" + }, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS" + } + + result = get_all_keys(sample_data) + + print(result) + misc ======== diff --git a/nested_lookup/__init__.py b/nested_lookup/__init__.py index 0ab3e32..367b245 100644 --- a/nested_lookup/__init__.py +++ b/nested_lookup/__init__.py @@ -1 +1 @@ -from .nested_lookup import nested_lookup +from .nested_lookup import nested_lookup, get_all_keys diff --git a/nested_lookup/nested_lookup.py b/nested_lookup/nested_lookup.py index 78ec193..b541fbb 100644 --- a/nested_lookup/nested_lookup.py +++ b/nested_lookup/nested_lookup.py @@ -2,20 +2,26 @@ from six import iteritems from collections import defaultdict + def nested_lookup(key, document, wild=False, with_keys=False): """Lookup a key in a nested document, return a list of values""" if with_keys: d = defaultdict(list) - for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + for k, v in _nested_lookup( + key, document, wild=wild, with_keys=with_keys + ): d[k].append(v) return d return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + def _nested_lookup(key, document, wild=False, with_keys=False): """Lookup a key in a nested document, yield a value""" if isinstance(document, list): for d in document: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): yield result if isinstance(document, dict): @@ -26,9 +32,39 @@ def _nested_lookup(key, document, wild=False, with_keys=False): else: yield v elif isinstance(v, dict): - for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + for result in _nested_lookup( + key, v, wild=wild, with_keys=with_keys + ): yield result elif isinstance(v, list): for d in v: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(dictionary): + for key, value in iteritems(dictionary): + if isinstance(value, dict): + result_list.append(key) + recrusion(dictionary=value) + elif isinstance(value, list): + result_list.append(key) + for list_items in value: + recrusion(dictionary=list_items) + else: + result_list.append(key) + + recrusion(dictionary=dictionary) + return result_list diff --git a/setup.py b/setup.py index 8e9b7d9..15bba6b 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ with open('requirements.txt', 'r') as f: setup( name = 'nested-lookup', - version = '0.1.5', + version = '0.1.6', description = 'lookup a key in a deeply nested document of dicts and lists', keywords = 'nested document dictionary dict list lookup schema json xml yaml', long_description = open('README.rst').read(), diff --git a/test_nested_loopkup.py b/test_nested_loopkup.py index 178335d..9dcc10b 100644 --- a/test_nested_loopkup.py +++ b/test_nested_loopkup.py @@ -1,18 +1,19 @@ from unittest import TestCase -from nested_lookup import nested_lookup +from nested_lookup import nested_lookup, get_all_keys + class TestNestedLookup(TestCase): def setUp(self): - self.subject_dict = {'a':1,'b':{'d':100},'c':{'d':200}} + self.subject_dict = {'a': 1, 'b': {'d': 100}, 'c': {'d': 200}} self.subject_dict2 = { - 'name' : 'Russell Ballestrini', - 'email_address' : 'test1@example.com', - 'other' : { - 'secondary_email' : 'test2@example.com', - 'EMAIL_RECOVERY' : 'test3@example.com', - 'email_address' : 'test4@example.com', + 'name': 'Russell Ballestrini', + 'email_address': 'test1@example.com', + 'other': { + 'secondary_email': 'test2@example.com', + 'EMAIL_RECOVERY': 'test3@example.com', + 'email_address': 'test4@example.com', }, } @@ -21,34 +22,34 @@ class TestNestedLookup(TestCase): self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100,200}, set(results)) + self.assertSetEqual({100, 200}, set(results)) def test_nested_lookup_wrapped_in_list(self): results = nested_lookup('d', [{}, self.subject_dict, {}]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100,200}, set(results)) + self.assertSetEqual({100, 200}, set(results)) def test_nested_lookup_wrapped_in_list_in_dict_in_list(self): - results = nested_lookup('d', [{}, {'H' : [self.subject_dict]} ]) + results = nested_lookup('d', [{}, {'H': [self.subject_dict]}]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100,200}, set(results)) + self.assertSetEqual({100, 200}, set(results)) def test_nested_lookup_wrapped_in_list_in_list(self): - results = nested_lookup('d', [ {}, [self.subject_dict, {}] ]) + results = nested_lookup('d', [{}, [self.subject_dict, {}]]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100,200}, set(results)) + self.assertSetEqual({100, 200}, set(results)) def test_wild_nested_lookup(self): results = nested_lookup( - key = 'mail', - document = self.subject_dict2 - wild = True, + key='mail', + document=self.subject_dict2, + wild=True, ) self.assertEqual(4, len(results)) self.assertIn('test1@example.com', results) @@ -57,21 +58,115 @@ class TestNestedLookup(TestCase): def test_wild_with_keys_nested_lookup(self): matches = nested_lookup( - key = 'mail', - document = self.subject_dict2, - wild = True, - with_keys = True, + key='mail', + document=self.subject_dict2, + wild=True, + with_keys=True, ) self.assertEqual(3, len(matches)) self.assertIn('email_address', matches) self.assertIn('secondary_email', matches) self.assertIn('EMAIL_RECOVERY', matches) - self.assertSetEqual({'test1@example.com','test4@example.com'}, set(matches['email_address'])) + self.assertSetEqual( + {'test1@example.com', 'test4@example.com'}, + set(matches['email_address']) + ) self.assertIn('test2@example.com', matches['secondary_email']) def test_nested_lookup_with_keys(self): matches = nested_lookup('d', self.subject_dict, with_keys=True) self.assertIn('d', matches) self.assertEqual(2, len(matches['d'])) - self.assertSetEqual({100,200}, set(matches['d'])) + self.assertSetEqual({100, 200}, set(matches['d'])) + +class TestGetAllKeys(TestCase): + + def setUp(self): + self.sample1 = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": { + "processor_name": "Intel Core i7", + "processor_speed": "2.7 GHz", + "core_details": { + "total_numberof_cores": "4", + "l2_cache(per_core)": "256 KB" + } + }, + "total_number_of_cores": "4", + "memory": "16 GB", + }, + "os_details": { + "product_version": "10.13.6", + "build_version": "17G65" + }, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS" + } + self.sample2 = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": [{ + "processor_name": "Intel Core i7", + "processor_speed": "2.7 GHz", + "core_details": { + "total_numberof_cores": "4", + "l2_cache(per_core)": "256 KB" + } + }], + "total_number_of_cores": "4", + "memory": "16 GB", + } + } + self.sample3 = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": [ + { + "processor_name": "Intel Core i7", + "processor_speed": "2.7 GHz", + }, + { + "total_numberof_cores": "4", + "l2_cache(per_core)": "256 KB" + } + ], + "total_number_of_cores": "4", + "memory": "16 GB", + } + } + + def test_sample_data1(self): + result = get_all_keys(self.sample1) + self.assertEqual(15, len(result)) + keys_to_verify = [ + 'model_name', 'core_details', 'l2_cache(per_core)', + 'build_version', 'date' + ] + for key in keys_to_verify: + self.assertIn(key, result) + + def test_sample_data2(self): + result = get_all_keys(self.sample2) + self.assertEqual(10, len(result)) + keys_to_verify = [ + 'hardware_details', 'processor_speed', + 'total_numberof_cores', 'memory' + ] + for key in keys_to_verify: + self.assertIn(key, result) + + def test_sample_data3(self): + result = get_all_keys(self.sample3) + self.assertEqual(9, len(result)) + keys_to_verify = [ + 'processor_details', 'processor_name', + 'l2_cache(per_core)', 'total_number_of_cores' + ] + for key in keys_to_verify: + self.assertIn(key, result) + + +if __name__ == '__main__': + pass From c1ca4c1e1e21c27aea7661235defd725343c65df Mon Sep 17 00:00:00 2001 From: Ramesh RV Date: Mon, 24 Sep 2018 12:23:56 +0530 Subject: [PATCH 2/2] Reverted PEP8 correction --- nested_lookup/nested_lookup.py | 18 +++---------- test_nested_loopkup.py | 47 ++++++++++++++++------------------ 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/nested_lookup/nested_lookup.py b/nested_lookup/nested_lookup.py index b541fbb..cc14075 100644 --- a/nested_lookup/nested_lookup.py +++ b/nested_lookup/nested_lookup.py @@ -2,26 +2,20 @@ from six import iteritems from collections import defaultdict - def nested_lookup(key, document, wild=False, with_keys=False): """Lookup a key in a nested document, return a list of values""" if with_keys: d = defaultdict(list) - for k, v in _nested_lookup( - key, document, wild=wild, with_keys=with_keys - ): + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): d[k].append(v) return d return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) - def _nested_lookup(key, document, wild=False, with_keys=False): """Lookup a key in a nested document, yield a value""" if isinstance(document, list): for d in document: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): yield result if isinstance(document, dict): @@ -32,15 +26,11 @@ def _nested_lookup(key, document, wild=False, with_keys=False): else: yield v elif isinstance(v, dict): - for result in _nested_lookup( - key, v, wild=wild, with_keys=with_keys - ): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): yield result elif isinstance(v, list): for d in v: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): yield result diff --git a/test_nested_loopkup.py b/test_nested_loopkup.py index 9dcc10b..2ab212f 100644 --- a/test_nested_loopkup.py +++ b/test_nested_loopkup.py @@ -6,14 +6,14 @@ from nested_lookup import nested_lookup, get_all_keys class TestNestedLookup(TestCase): def setUp(self): - self.subject_dict = {'a': 1, 'b': {'d': 100}, 'c': {'d': 200}} + self.subject_dict = {'a':1,'b':{'d':100},'c':{'d':200}} self.subject_dict2 = { - 'name': 'Russell Ballestrini', - 'email_address': 'test1@example.com', - 'other': { - 'secondary_email': 'test2@example.com', - 'EMAIL_RECOVERY': 'test3@example.com', - 'email_address': 'test4@example.com', + 'name' : 'Russell Ballestrini', + 'email_address' : 'test1@example.com', + 'other' : { + 'secondary_email' : 'test2@example.com', + 'EMAIL_RECOVERY' : 'test3@example.com', + 'email_address' : 'test4@example.com', }, } @@ -22,34 +22,34 @@ class TestNestedLookup(TestCase): self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100, 200}, set(results)) + self.assertSetEqual({100,200}, set(results)) def test_nested_lookup_wrapped_in_list(self): results = nested_lookup('d', [{}, self.subject_dict, {}]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100, 200}, set(results)) + self.assertSetEqual({100,200}, set(results)) def test_nested_lookup_wrapped_in_list_in_dict_in_list(self): - results = nested_lookup('d', [{}, {'H': [self.subject_dict]}]) + results = nested_lookup('d', [{}, {'H' : [self.subject_dict]} ]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100, 200}, set(results)) + self.assertSetEqual({100,200}, set(results)) def test_nested_lookup_wrapped_in_list_in_list(self): - results = nested_lookup('d', [{}, [self.subject_dict, {}]]) + results = nested_lookup('d', [ {}, [self.subject_dict, {}] ]) self.assertEqual(2, len(results)) self.assertIn(100, results) self.assertIn(200, results) - self.assertSetEqual({100, 200}, set(results)) + self.assertSetEqual({100,200}, set(results)) def test_wild_nested_lookup(self): results = nested_lookup( - key='mail', - document=self.subject_dict2, - wild=True, + key = 'mail', + document = self.subject_dict2 + wild = True, ) self.assertEqual(4, len(results)) self.assertIn('test1@example.com', results) @@ -58,26 +58,23 @@ class TestNestedLookup(TestCase): def test_wild_with_keys_nested_lookup(self): matches = nested_lookup( - key='mail', - document=self.subject_dict2, - wild=True, - with_keys=True, + key = 'mail', + document = self.subject_dict2, + wild = True, + with_keys = True, ) self.assertEqual(3, len(matches)) self.assertIn('email_address', matches) self.assertIn('secondary_email', matches) self.assertIn('EMAIL_RECOVERY', matches) - self.assertSetEqual( - {'test1@example.com', 'test4@example.com'}, - set(matches['email_address']) - ) + self.assertSetEqual({'test1@example.com','test4@example.com'}, set(matches['email_address'])) self.assertIn('test2@example.com', matches['secondary_email']) def test_nested_lookup_with_keys(self): matches = nested_lookup('d', self.subject_dict, with_keys=True) self.assertIn('d', matches) self.assertEqual(2, len(matches['d'])) - self.assertSetEqual({100, 200}, set(matches['d'])) + self.assertSetEqual({100,200}, set(matches['d'])) class TestGetAllKeys(TestCase):