diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..772f7edf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +language: python +python: + - "2.6" + - "2.7" +env: + - DJANGO_VERSION=1.2 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.2 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.2 REDIS_VERSION=2.6 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.6 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 +install: + - pip install Django==$DJANGO_VERSION + - pip install -e git://github.com/sebleier/redis-py.git#egg=redis-py + - pip install hiredis + - ./install_redis.sh +branches: + only: + - master + - experimental +script: ./travis_run_all_tests.sh \ No newline at end of file diff --git a/install_redis.sh b/install_redis.sh new file mode 100755 index 00000000..22cfe7b6 --- /dev/null +++ b/install_redis.sh @@ -0,0 +1,9 @@ +if [ -z $REDIS_VERSION ]; then + echo "This script is used to install redis for Travis-CI" +else + cd .. + git clone https://github.com/antirez/redis.git + cd redis + git checkout $REDIS_VERSION + make +fi \ No newline at end of file diff --git a/redis_cache/cache.py b/redis_cache/cache.py index 055073d6..01077247 100644 --- a/redis_cache/cache.py +++ b/redis_cache/cache.py @@ -188,6 +188,23 @@ def get(self, key, default=None, version=None): result = self.unpickle(value) return result + def hget(self, name, key, default=None, version=None): + """ + Retrieve a value from the cache as hash. + + Returns unpickled value if key is found, the default if not. + """ + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + value = self._client.hget(name, key) + if value is None: + return default + try: + result = int(value) + except (ValueError, TypeError): + result = self.unpickle(value) + return result + def _set(self, key, value, timeout, client): if timeout == 0: return client.set(key, value) @@ -196,6 +213,15 @@ def _set(self, key, value, timeout, client): else: return False + def _hset(self, name, key, value, timeout, client): + """Set value in hash""" + # Redis call + status = client.hset( name, key, value) + if timeout > 0: + # Set expire time + client.expire(name, timeout) + return status + def set(self, key, value, timeout=None, version=None, client=None): """ Persist a value to the cache, and set an optional expiration time. @@ -217,6 +243,30 @@ def set(self, key, value, timeout=None, version=None, client=None): # result is a boolean return result + def hset(self, name, key, value, timeout=None, version=None, client=None): + """ + Sets field in the hash stored at key to value + and set an optional expiration time. + """ + if not client: + client = self._client + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + if timeout is None: + # To store it persistently + timeout = 0 + try: + value = float(value) + # If you lose precision from the typecast to str, then pickle value + if int(value) != value: + raise TypeError + except (ValueError, TypeError): + result = self._hset(name, key, pickle.dumps(value), int(timeout), client) + else: + result = self._hset(name, key, int(value), int(timeout), client) + # result is a boolean + return result + def delete(self, key, version=None): """ Remove a key from the cache. @@ -267,6 +317,31 @@ def get_many(self, keys, version=None): recovered_data[map_keys[key]] = value return recovered_data + def hget_many(self, name, keys, version=None): + """ + Retrieve many keys from hash. + Keys is the list of the key in the hash. + Result is returned as key value pair dict. + """ + recovered_data = SortedDict() + new_keys = map(lambda key: self.make_key(key, version=version), keys) + map_keys = dict(zip(new_keys, keys)) + name = self.make_key(name, version=version) + # Redis call + results = self._client.hmget(name, new_keys) + # Iterate to convert the result into proper format. + for key, value in zip(new_keys, results): + if value is None: + continue + try: + value = int(value) + except (ValueError, TypeError): + value = self.unpickle(value) + if isinstance(value, basestring): + value = smart_unicode(value) + recovered_data[map_keys[key]] = value + return recovered_data + def set_many(self, data, timeout=None, version=None): """ Set a bunch of values in the cache at once from a dict of key/value @@ -280,6 +355,38 @@ def set_many(self, data, timeout=None, version=None): self.set(key, value, timeout, version=version, client=pipeline) pipeline.execute() + + def hset_many(self, name, mapping_dict, timeout=None, version=None): + """ + Set a bunch of values in the cache at once from a dict of key/value + pairs. This is much more efficient than calling hset() multiple times. + + If timeout is given, that timeout will be used for the key; otherwise + stored persistently + """ + # Convert the keys to version format + mapping_dict = dict( (self.make_key(key, version), value) for key, value in mapping_dict.items() ) + name = self.make_key(name, version=version) + # Iterate to convert the values to appropriate format. + for key, value in mapping_dict.items(): + try: + value = float(value) + # If you lose precision from the typecast to str, then pickle value + if int(value) != value: + raise TypeError + except (ValueError, TypeError): + value = pickle.dumps(value) + else: + value = int(value) + mapping_dict[key] = value + + # Redis call + result = self._client.hmset(name, mapping_dict) + if timeout is not None: + # Set timeout of the name + self._client.expire(name, timeout) + return result + def incr(self, key, delta=1, version=None): """ Add delta to value in the cache. If the key does not exist, raise a @@ -296,6 +403,33 @@ def incr(self, key, delta=1, version=None): self.set(key, value) return value + def hincr(self, name, key, delta=1, version=None): + """ + Add delta to value in the cache of the hash. + If the key does not exist, raise a + ValueError exception. + """ + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + exists = self._client.hexists(name, key) + if not exists: + raise ValueError("Key '%s' not found" % key) + try: + # Redis call + value = self._client.hincrby(name, key, delta) + except redis.ResponseError: + value = self.hget(name, key) + 1 + # Redis call + self.hset(name, key, value) + return value + + def has_hkey(self, name, key, version=None): + """ + The name and key exist in the hash + """ + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + return self._client.hexists(name, key) class RedisCache(CacheClass): """ diff --git a/run_all_tests.sh b/run_all_tests.sh new file mode 100755 index 00000000..ba38bf69 --- /dev/null +++ b/run_all_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./run_tests.py --settings=tests.settings +./run_tests.py --settings=tests.python_parser_settings +./run_tests.py --settings=tests.sockets_settings \ No newline at end of file diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 00000000..3b43fceb --- /dev/null +++ b/run_tests.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +from __future__ import with_statement +from optparse import OptionParser +import os +from os.path import dirname, abspath, join +import sys +from django.conf import settings +from django.template import Template, Context +from django.utils import importlib +from redis.server import server + + +def load_settings(module): + try: + mod = importlib.import_module(module) + except (ImportError): + return None + + conf = {} + for setting in dir(mod): + if setting == setting.upper(): + conf[setting] = getattr(mod, setting) + return conf + + +class TmpFile(object): + def __init__(self, path, contents): + self.path =path + self.contents = contents + + def __enter__(self): + self.file = open(self.path, "w") + self.file.write(self.contents) + self.file.close() + + def __exit__(self, exc_type, exc_value, traceback): + os.remove(self.path) + + +def runtests(options): + os.environ['DJANGO_SETTINGS_MODULE'] = options.settings + + conf = load_settings(options.settings) + + if conf is None: + sys.stderr.write('Cannot load settings module: %s\n' % options.settings) + return sys.exit(1) + + settings.configure(**conf) + + redis_conf_path = options.conf or join(dirname(__file__), 'tests', 'redis.conf') + + server.configure(options.server_path, redis_conf_path, 0) + + try: + redis_conf_template = open(join(dirname(__file__), 'tests' ,'redis.conf.tpl')).read() + except OSError, IOError: + sys.stderr.write('Cannot find template for redis.conf.\n') + context = Context({ + 'redis_socket': join(dirname(abspath(__file__)), 'tests', 'redis.sock') + }) + + contents = Template(redis_conf_template).render(context) + + with TmpFile(redis_conf_path, contents): + with server: + from django.test.simple import DjangoTestSuiteRunner + runner = DjangoTestSuiteRunner(verbosity=options.verbosity, interactive=True, failfast=False) + failures = runner.run_tests(['testapp']) + + sys.exit(failures) + + +if __name__ == '__main__': + parser = OptionParser() + parser.add_option("-s", "--server", dest="server_path", action="store", + type="string", default=None, help="Path to the redis server executable") + parser.add_option("-c", "--conf", dest="conf", default=None, + help="Path to the redis configuration file.") + parser.add_option("-v", "--verbosity", dest="verbosity", default=1, type="int", + help="Change the verbostiy of the redis-server.") + parser.add_option("--settings", dest="settings", default="tests.settings", + help="Django settings module to use for the tests.") + + (options, args) = parser.parse_args() + + parent = dirname(abspath(__file__)) + sys.path.insert(0, parent) + + runtests(options) diff --git a/sockettests.py b/sockettests.py deleted file mode 100755 index 57479456..00000000 --- a/sockettests.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import sys -from os.path import dirname, abspath -from django.conf import settings - - -cache_settings = { - 'DATABASES': { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } - }, - 'INSTALLED_APPS': [ - 'tests.testapp', - ], - 'ROOT_URLCONF': 'tests.urls', - 'CACHES': { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '/tmp/redis.sock', - 'OPTIONS': { - 'DB': 15, - 'PASSWORD': 'yadayada', - 'PARSER_CLASS': 'redis.connection.HiredisParser' - }, - }, - }, -} - -if not settings.configured: - settings.configure(**cache_settings) - -from django.test.simple import DjangoTestSuiteRunner - -def runtests(*test_args): - if not test_args: - test_args = ['testapp'] - parent = dirname(abspath(__file__)) - sys.path.insert(0, parent) - runner = DjangoTestSuiteRunner(verbosity=1, interactive=True, failfast=False) - failures = runner.run_tests(test_args) - sys.exit(failures) - -if __name__ == '__main__': - runtests(*sys.argv[1:]) diff --git a/tcptests.py b/tcptests.py deleted file mode 100755 index 74234725..00000000 --- a/tcptests.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -import sys -from os.path import dirname, abspath -from django.conf import settings - - -cache_settings = { - 'DATABASES': { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } - }, - 'INSTALLED_APPS': [ - 'tests.testapp', - ], - 'ROOT_URLCONF': 'tests.urls', - 'CACHES': { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '127.0.0.1:6379', - 'OPTIONS': { - 'DB': 15, - 'PASSWORD': 'yadayada', - 'PARSER_CLASS': 'redis.connection.HiredisParser' - }, - }, - }, -} - - -if not settings.configured: - settings.configure(**cache_settings) - -from django.test.simple import DjangoTestSuiteRunner - -def runtests(*test_args): - if not test_args: - test_args = ['testapp'] - parent = dirname(abspath(__file__)) - sys.path.insert(0, parent) - runner = DjangoTestSuiteRunner(verbosity=1, interactive=True, failfast=False) - failures = runner.run_tests(test_args) - sys.exit(failures) - -if __name__ == '__main__': - runtests(*sys.argv[1:]) diff --git a/tests/base_settings.py b/tests/base_settings.py new file mode 100644 index 00000000..3ca25ad0 --- /dev/null +++ b/tests/base_settings.py @@ -0,0 +1,10 @@ +DEBUG = True +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + } +} +INSTALLED_APPS = [ + 'tests.testapp', +] +ROOT_URLCONF = 'tests.urls' diff --git a/tests/benchmark.py b/tests/benchmark.py index 8ee50f66..5af02f37 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -8,7 +8,6 @@ python benchmark.py master python benchamrk.py some-branch """ - import os import sys from time import time @@ -48,12 +47,16 @@ def tearDown(self): pass def timetrial(self): + self.cache = cache.get_cache('default') + # Initializing the redis to basic settings + self.cache._client.flushall() self.setUp() start = time() self.run() t = time() - start self.tearDown() - return t + final_usuage = self.cache._client.info()['used_memory_human'] + return t, final_usuage def run(self): pass @@ -63,7 +66,7 @@ def run_benchmarks(cls): for benchmark in cls.benchmarks: benchmark = benchmark() print benchmark.__doc__ - print "Time: %s" % (benchmark.timetrial()) + print "Time: %s; Memory: %s" % (benchmark.timetrial()) class GetAndSetBenchmark(Benchmark): @@ -96,6 +99,8 @@ def setUp(self): self.values[h(h(i))] = h(i) self.ints.append(i) self.strings.append(h(i)) + for k, v in self.values.items(): + self.cache.set(k, v) def run(self): for i in self.ints: @@ -116,6 +121,42 @@ def run(self): self.cache.set_many(self.values) value = self.cache.get_many(self.values.keys()) +class HGetAndHSetBenchmark(Benchmark): + "Settings and Getting Mixed for Hashes" + + def setUp(self): + self.cache = cache.get_cache('default') + self.values = {} + # 100*300 = 30000 + for i in range(100): + self.values[h(i)] = i + self.values[h(h(i))] = h(i) + + def run(self): + for k, v in self.values.items(): + for i in range(300): + self.cache.hset(k, str(i), v) + for k, v in self.values.items(): + for i in range(300): + self.cache.hget(k, str(i)) + +class HMsetAndHMGet(Benchmark): + "Getting and setting many mixed values for Hashes" + + def setUp(self): + self.cache = cache.get_cache('default') + self.values = {} + for i in range(100): + self.values[h(i)] = i + self.values[h(h(i))] = h(i) + + def run(self): + for k, v in self.values.items(): + mapping_dict = dict( (str(i), v) for i in range(300) ) + self.cache.hset_many(k, mapping_dict) + for k, v in self.values.items(): + many_keys = [ str(i) for i in range(300) ] + self.cache.hget_many(k, many_keys) if __name__ == "__main__": - Benchmark.run_benchmarks() \ No newline at end of file + Benchmark.run_benchmarks() diff --git a/tests/settings.py b/tests/hiredis_parser_settings.py similarity index 56% rename from tests/settings.py rename to tests/hiredis_parser_settings.py index 22c26f94..002c7d73 100644 --- a/tests/settings.py +++ b/tests/hiredis_parser_settings.py @@ -1,14 +1,5 @@ -DEBUG = True +from .base_settings import * -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } -} - -INSTALLED_APPS = [ - 'tests.testapp', -] CACHES = { 'default': { @@ -17,8 +8,7 @@ 'OPTIONS': { # optional 'DB': 15, 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.HiredisParser', }, }, } - -ROOT_URLCONF = 'tests.urls' diff --git a/tests/python_parser_settings.py b/tests/python_parser_settings.py new file mode 100644 index 00000000..98de7e08 --- /dev/null +++ b/tests/python_parser_settings.py @@ -0,0 +1,14 @@ +from .base_settings import * + + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': '127.0.0.1:6379', + 'OPTIONS': { + 'DB': 15, + 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.PythonParser' + }, + }, +} diff --git a/tests/redis.conf.tpl b/tests/redis.conf.tpl new file mode 100644 index 00000000..764c2a4e --- /dev/null +++ b/tests/redis.conf.tpl @@ -0,0 +1,525 @@ +# Redis configuration file example + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /var/run/redis.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 6380 + +# If you want you can bind a single interface, if the bind option is not +# specified all the interfaces will listen for incoming connections. +# +# bind 127.0.0.1 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +unixsocket {{ redis_socket }} +#unixsocketperm 755 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# Set server verbosity to 'debug' +# it can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also 'stdout' can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile stdout + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################# +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving at all commenting all the "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in an hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# distater will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usually even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since verison 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# Also the Append Only File will be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir ./ + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. Note that the configuration is local to the slave +# so for example it is possible to configure the slave to save the DB with a +# different interval, or to listen to another port, and so on. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave lost the connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extend you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets a timeout for both Bulk transfer I/O timeout and +# master data or ping response timeout. The default value is 60 seconds. +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# of hard to guess so that it will be still available for internal-use +# tools but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command renaming it into +# an empty string: +# +# rename-command CONFIG "" + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able ot configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# accordingly to the eviction policy selected (see maxmemmory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# an hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached? You can select among five behavior: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key accordingly to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with all the kind of policies, Redis will return an error on write +# operations, when there are not suitable keys for eviction. +# +# At the date of writing this commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") +# appendfilename appendonly.aof + +# The fsync() call tells the Operating System to actually write data on disk +# instead to wait for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log . Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec" that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving the durability of Redis is +# the same as "appendfsync none", that in practical terms means that it is +# possible to lost up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size will growth by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (or if no rewrite happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceed the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write commands was +# already issue by the script but the user don't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happens to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into an hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# active rehashing the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply form time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients +# slave -> slave clients and MONITOR clients +# pubsub -> clients subcribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled just setting it to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis server but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# include /path/to/local.conf +# include /path/to/other.conf diff --git a/tests/sockets_settings.py b/tests/sockets_settings.py new file mode 100644 index 00000000..855800bf --- /dev/null +++ b/tests/sockets_settings.py @@ -0,0 +1,16 @@ +from .base_settings import * +from os.path import join, dirname + + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': join(dirname(__file__), 'redis.sock'), + 'OPTIONS': { # optional + 'DB': 15, + 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.HiredisParser', + }, + }, +} + diff --git a/tests/testapp/tests.py b/tests/testapp/tests.py index 69a188da..64bcbac3 100644 --- a/tests/testapp/tests.py +++ b/tests/testapp/tests.py @@ -311,7 +311,7 @@ def test_incr_version(self): self.assertEqual(self.cache.get(new_key), 'spam') def test_incr_with_pickled_integer(self): - "Testing case where there exists a pickled integer and we increment it" + #Testing case where there exists a pickled integer and we increment it number = 42 key = self.cache.make_key("key") @@ -356,6 +356,92 @@ def test_multiple_connection_pool_connections(self): c3 = get_cache('redis_cache.cache://127.0.0.1:6379?db=15') self.assertEqual(len(pool._connection_pools), 2) + def test_hset(self): + # Set in a hash + self.cache.hset('a', 'a1', 'A') + self.assertEqual(self.cache.hget('a','a1'),'A') + + def test_float_hcaching(self): + # Set float value in a hash + self.cache.hset('a', 'a1', 1.1) + a = self.cache.hget('a', 'a1') + self.assertEqual(a, 1.1) + + def test_string_float_hcaching(self): + self.cache.hset('a', 'a1', '1.1') + a = self.cache.hget('a','a1') + self.assertEqual(a, 1.1) + + def test_hget_many(self): + # Multiple cache keys can be returned using get_many + self.cache.hset('a', 'a1', 'A1') + self.cache.hset('a', 'a2', 'A2') + self.cache.hset('a', 'a3', 'A3') + self.assertEqual(self.cache.hget_many('a',['a1','a2']), {'a1': 'A1', 'a2': 'A2'}) + self.assertEqual(self.cache.hget_many('a',['a3']), {'a3': 'A3'}) + + def test_hset_many(self): + # Multiple cache keys can be set using hset_many in one call + self.cache.hset_many('a', {'a1':'A1', 'a2':'A2', 'a3':'A3'}) + self.assertEqual(self.cache.hget_many('a',['a1','a2']), {'a1': 'A1', 'a2': 'A2'}) + self.assertEqual(self.cache.hget('a','a1'), 'A1') + + def test_hexpiration(self): + # Cache values can be set to expire + self.cache.hset('expire1', 'key1', 1, 1) + self.cache.hset('expire2', 'key2', 1, 1) + self.cache.hset('expire3', 'key3', 1, 1) + + time.sleep(2) + self.assertEqual(self.cache.hget("expire1","key1"), None) + + self.cache.hset("expire2", "key2", "newvalue") + self.assertEqual(self.cache.hget("expire2", "key2"), "newvalue") + + def test_hset_expiration_timeout_None(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value) + self.assertTrue(self.cache._client.ttl(key) is None) + + def test_hset_expiration_timeout_zero(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value, timeout=0) + self.assertTrue(self.cache._client.ttl(key) is None) + self.assertTrue(self.cache.has_hkey(key,'a')) + + def test_hset_expiration_timeout_negative(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value, timeout=-1) + self.assertTrue(self.cache._client.ttl(key) is None) + + def test_hash_unicode(self): + # Unicode values can be cached + stuff = { + u'ascii': (u'key1',u'ascii_value'), + u'unicode_ascii': (u'key2',u'Iñtërnâtiônàlizætiøn1'), + u'Iñtërnâtiônàlizætiøn': (u'key3',u'Iñtërnâtiônàlizætiøn2'), + u'ascii': (u'key4',{u'x' : 1 }) + } + for (key, value) in stuff.items(): + self.cache.hset(key, value[0], value[1]) + self.assertEqual(self.cache.hget(key, value[0]), value[1]) + + def test_hash_binary_string(self): + # Binary strings should be cachable + from zlib import compress, decompress + value = 'value_to_be_compressed' + compressed_value = compress(value) + self.cache.hset('binary1', 'b1', compressed_value) + compressed_result = self.cache.hget('binary1','b1') + self.assertEqual(compressed_value, compressed_result) + self.assertEqual(value, decompress(compressed_result)) + + def test_has_hkey(self): + # Presence of hash key + self.cache.hset('A','A1',100) + self.assertEqual(self.cache.has_hkey('A','A1'),True) + self.assertEqual(self.cache.has_hkey('B','B1'),False) + self.assertEqual(self.cache.has_hkey('A','A10'),False) if __name__ == '__main__': unittest.main() diff --git a/travis_run_all_tests.sh b/travis_run_all_tests.sh new file mode 100755 index 00000000..59e82551 --- /dev/null +++ b/travis_run_all_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./run_tests.py --settings=tests.settings -s ../redis/src/redis-server +./run_tests.py --settings=tests.python_parser_settings -s ../redis/src/redis-server +./run_tests.py --settings=tests.sockets_settings -s ../redis/src/redis-server \ No newline at end of file