Merge pull request #32 from Azure/python2to3

[sonic-dbsyncd] Changes to convert sonic-dbsyncd from python 2 to 3

Author: abdosi

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -2,11 +2,12 @@
 
 dependencies = [
     'swsssdk>=1.3.0',
-    'enum34>=1.1.6',
 ]
 
 test_deps = [
-    'mockredispy>=2.9.3',
+    'pytest',
+    'mock>=2.0.0',
+    'mockredispy>=2.9.3'
 ]
 
 high_performance_deps = [
@@ -17,9 +18,9 @@
     name='sonic-d',
     install_requires=dependencies,
     version='2.0.0',
+    tests_require=test_deps,
     packages=find_packages('src'),
     extras_require={
-        'testing': test_deps,
         'high_perf': high_performance_deps,
     },
     license='Apache 2.0',
@@ -31,10 +32,13 @@
         'sonic_syncd': 'src/sonic_syncd',
         'lldp_syncd': 'src/lldp_syncd',
     },
-    classifiers=[
+    setup_requires= [
+        'pytest-runner',
+        'wheel',
+    ],
+    classifiers = [
         'Intended Audience :: Developers',
         'Operating System :: Linux',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.7',
     ],
 )

src/lldp_syncd/daemon.py

@@ -124,7 +124,7 @@ def get_sys_capability_list(self, if_attributes, if_name, chassis_id):
             if 'capability' in if_attributes['chassis']:
                 capability_list = if_attributes['chassis']['capability']
             else:
-                capability_list = if_attributes['chassis'].values()[0]['capability']
+                capability_list = list(if_attributes['chassis'].values())[0]['capability']
             # {'enabled': ..., 'type': 'capability'}
             if isinstance(capability_list, dict):
                 capability_list = [capability_list]
@@ -233,7 +233,7 @@ def parse_update(self, lldp_json):
                     rem_port_keys = ('lldp_rem_port_id_subtype',
                                      'lldp_rem_port_id',
                                      'lldp_rem_port_desc')
-                    parsed_port = zip(rem_port_keys, self.parse_port(if_attributes['port']))
+                    parsed_port = list(zip(rem_port_keys, self.parse_port(if_attributes['port'])))
                     parsed_interfaces[if_name].update(parsed_port)
 
                 chassis_id = ''
@@ -244,8 +244,8 @@ def parse_update(self, lldp_json):
                                         'lldp_rem_sys_name',
                                         'lldp_rem_sys_desc',
                                         'lldp_rem_man_addr')
-                    parsed_chassis = zip(rem_chassis_keys,
-                                         self.parse_chassis(if_attributes['chassis']))
+                    parsed_chassis = list(zip(rem_chassis_keys,
+                                         self.parse_chassis(if_attributes['chassis'])))
                     parsed_interfaces[if_name].update(parsed_chassis)
                     chassis_id = parsed_chassis[1][1]
 
@@ -296,7 +296,7 @@ def parse_chassis(self, chassis_attributes):
                 attributes = chassis_attributes
                 id_attributes = chassis_attributes['id']
             else:
-                (sys_name, attributes) = chassis_attributes.items()[0]
+                (sys_name, attributes) = list(chassis_attributes.items())[0]
                 id_attributes = attributes.get('id', '')
 
             chassis_id_subtype = str(self.ChassisIdSubtypeMap[id_attributes['type']].value)
@@ -352,7 +352,7 @@ def sync(self, parsed_update):
         logger.debug("Initiating LLDPd sync to Redis...")
 
         # push local chassis data to APP DB
-        if parsed_update.has_key('local-chassis'):
+        if 'local-chassis' in parsed_update:
             chassis_update = parsed_update.pop('local-chassis')
             if chassis_update != self.chassis_cache:
                 self.db_connector.delete(self.db_connector.APPL_DB,

tests/mock_tables/dbconnector.py

@@ -1,10 +1,24 @@
 # MONKEY PATCH!!!
 import json
 import os
+import sys
 
 import mockredis
 import swsssdk.interface
 from swsssdk.interface import redis
+from swsssdk import SonicV2Connector
+
+if sys.version_info >= (3, 0):
+    long = int
+    xrange = range
+    basestring = str
+
+_old_connect_SonicV2Connector = SonicV2Connector.connect
+
+def connect_SonicV2Connector(self, db_name, retry_on=True):
+    self.dbintf.redis_kwargs['db_name'] = db_name
+    self.dbintf.redis_kwargs['decode_responses'] = True
+    _old_connect_SonicV2Connector(self, db_name, retry_on)
 
 
 def _subscribe_keyspace_notification(self, db_name, client):
@@ -38,6 +52,7 @@ class SwssSyncClient(mockredis.MockRedis):
     def __init__(self, *args, **kwargs):
         super(SwssSyncClient, self).__init__(strict=True, *args, **kwargs)
         db = kwargs.pop('db')
+        self.decode_responses = kwargs.pop('decode_responses', False) == True
 
         self.pubsub = MockPubSub()
 
@@ -55,6 +70,17 @@ def __init__(self, *args, **kwargs):
                     for k, v in table.items():
                         self.hset(h, k, v)
 
+    # Patch mockredis/mockredis/client.py
+    # The offical implementation assume decode_responses=False
+    # Here we detect the option and decode after doing encode
+    def _encode(self, value):
+        "Return a bytestring representation of the value. Taken from redis-py connection.py"
+
+        value = super(SwssSyncClient, self)._encode(value)
+
+        if self.decode_responses:
+           return value.decode('utf-8')
+    
     # Patch mockredis/mockredis/client.py
     # The official implementation will filter out keys with a slash '/'
     # ref: https://github.com/locationlabs/mockredis/blob/master/mockredis/client.py
@@ -76,9 +102,10 @@ def keys(self, pattern='*'):
         regex = re.compile(regex)
 
         # Find every key that matches the pattern
-        return [key for key in self.redis.keys() if regex.match(key.decode('utf-8'))]
+        return [key for key in self.redis.keys() if regex.match(key)]
 
 
 swsssdk.interface.DBInterface._subscribe_keyspace_notification = _subscribe_keyspace_notification
 mockredis.MockRedis.config_set = config_set
 redis.StrictRedis = SwssSyncClient
+SonicV2Connector.connect = connect_SonicV2Connector

tests/subproc_outputs/lldpctl.json

@@ -136,6 +136,7 @@
               "type": "ifname",
               "value": "Ethernet1"
             },
+            "descr": "this is a port description",
             "mfs": "9236"
           },
           "via": "LLDP",

tests/test_lldpSyncDaemon.py

@@ -79,12 +79,11 @@ def test_sync_roundtrip(self):
         print(json.dumps(dump, indent=3))
 
         # convert dict keys to ints for easy comparison
-        jo = {int(re.findall(r'\d+', k)[0]): v for k, v in parsed_update.items()}
-        r_out = {int(re.findall(r'\d+', k)[0]): v for k, v in dump.items()}
-        self.assertEqual(jo, r_out)
+        jo = {'LLDP_ENTRY_TABLE:'+ k: v for k, v in parsed_update.items()}
+        self.assertEqual(jo, dump)
 
         # test enumerations
-        for k, v in r_out.items():
+        for k, v in dump.items():
             chassis_subtype = v['lldp_rem_chassis_id_subtype']
             chassis_id = v['lldp_rem_chassis_id']
             if int(chassis_subtype) == lldp_syncd.conventions.LldpChassisIdSubtype.macAddress:
@@ -94,22 +93,22 @@ def test_sync_roundtrip(self):
                 self.fail("Test data only contains chassis MACs")
 
     def test_timeparse(self):
-        self.assertEquals(lldp_syncd.daemon.parse_time("0 day, 05:09:02"), make_seconds(0, 5, 9, 2))
-        self.assertEquals(lldp_syncd.daemon.parse_time("2 days, 05:59:02"), make_seconds(2, 5, 59, 2))
+        self.assertEqual(lldp_syncd.daemon.parse_time("0 day, 05:09:02"), make_seconds(0, 5, 9, 2))
+        self.assertEqual(lldp_syncd.daemon.parse_time("2 days, 05:59:02"), make_seconds(2, 5, 59, 2))
 
     def parse_mgmt_ip(self, json_file):
         parsed_update = self.daemon.parse_update(json_file)
         mgmt_ip_str = parsed_update['local-chassis'].get('lldp_loc_man_addr')
         json_chassis = json.dumps(json_file['lldp_loc_chassis']['local-chassis']['chassis'])
         chassis_dict = json.loads(json_chassis)
-        json_mgmt_ip = chassis_dict.values()[0]['mgmt-ip']
+        json_mgmt_ip = list(chassis_dict.values())[0]['mgmt-ip']
         if isinstance(json_mgmt_ip, list):
             i=0
             for mgmt_ip in mgmt_ip_str.split(','):
-                self.assertEquals(mgmt_ip, json_mgmt_ip[i])
+                self.assertEqual(mgmt_ip, json_mgmt_ip[i])
                 i+=1
         else:
-            self.assertEquals(mgmt_ip_str, json_mgmt_ip)
+            self.assertEqual(mgmt_ip_str, json_mgmt_ip)
 
     def test_multiple_mgmt_ip(self):
         self.parse_mgmt_ip(self._json)
@@ -123,7 +122,7 @@ def test_loc_chassis(self):
         self.daemon.sync(parsed_update)
         db = create_dbconnector()
         db_loc_chassis_data = db.get_all(db.APPL_DB, 'LLDP_LOC_CHASSIS')
-        self.assertEquals(parsed_loc_chassis, db_loc_chassis_data)
+        self.assertEqual(parsed_loc_chassis, db_loc_chassis_data)
 
     def test_remote_sys_capability_list(self):
         interface_list = self._interface_only['lldp'].get('interface')