From e275e29f1f86e99da853f2ad480645d5d8670eb8 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 15:21:24 -0700 Subject: [PATCH 01/15] First stab at getting travis-ci to work. --- .travis.yml | 26 + install_redis.sh | 9 + run_all_tests.sh | 5 + run_tests.py | 89 +++ sockettests.py | 45 -- tcptests.py | 46 -- tests/base_settings.py | 10 + ...settings.py => hiredis_parser_settings.py} | 14 +- tests/python_parser_settings.py | 14 + tests/redis.conf.tpl | 525 ++++++++++++++++++ tests/sockets_settings.py | 16 + travis_run_all_tests.sh | 5 + 12 files changed, 701 insertions(+), 103 deletions(-) create mode 100644 .travis.yml create mode 100755 install_redis.sh create mode 100755 run_all_tests.sh create mode 100755 run_tests.py delete mode 100755 sockettests.py delete mode 100755 tcptests.py create mode 100644 tests/base_settings.py rename tests/{settings.py => hiredis_parser_settings.py} (56%) create mode 100644 tests/python_parser_settings.py create mode 100644 tests/redis.conf.tpl create mode 100644 tests/sockets_settings.py create mode 100755 travis_run_all_tests.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..89b1afef --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +python: + - "2.5" + - "2.6" + - "2.7" +env: + #- DJANGO_VERSION=1.2 REDIS_VERSION=2.2 + #- DJANGO_VERSION=1.2 REDIS_VERSION=2.4 + #- DJANGO_VERSION=1.2 REDIS_VERSION=2.6 + #- DJANGO_VERSION=1.3 REDIS_VERSION=2.2 + #- DJANGO_VERSION=1.3 REDIS_VERSION=2.4 + #- DJANGO_VERSION=1.3 REDIS_VERSION=2.6 + #- DJANGO_VERSION=1.4 REDIS_VERSION=2.2 + #- DJANGO_VERSION=1.4 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 +install: + - pip install -q Django==$DJANGO_VERSION + - pip install -q -e git://github.com/sebleier/redis-py.git#egg=redis-py + - install_redis.sh +branches: + only: + - master + - experimental + + +script: travis_run_all_tests \ No newline at end of file diff --git a/install_redis.sh b/install_redis.sh new file mode 100755 index 00000000..22cfe7b6 --- /dev/null +++ b/install_redis.sh @@ -0,0 +1,9 @@ +if [ -z $REDIS_VERSION ]; then + echo "This script is used to install redis for Travis-CI" +else + cd .. + git clone https://github.com/antirez/redis.git + cd redis + git checkout $REDIS_VERSION + make +fi \ No newline at end of file diff --git a/run_all_tests.sh b/run_all_tests.sh new file mode 100755 index 00000000..ba38bf69 --- /dev/null +++ b/run_all_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./run_tests.py --settings=tests.settings +./run_tests.py --settings=tests.python_parser_settings +./run_tests.py --settings=tests.sockets_settings \ No newline at end of file diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 00000000..9a435b6c --- /dev/null +++ b/run_tests.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +from optparse import OptionParser +import os +from os.path import dirname, abspath, join +import sys +from django.conf import settings +from django.template import Template, Context +from django.utils import importlib +from redis.server import server + + +def load_settings(module): + try: + mod = importlib.import_module(module) + except (ImportError): + return None + + conf = {} + for setting in dir(mod): + if setting == setting.upper(): + conf[setting] = getattr(mod, setting) + return conf + + +class TmpFile(object): + def __init__(self, path, contents): + self.path =path + self.contents = contents + + def __enter__(self): + self.file = open(self.path, "w") + self.file.write(self.contents) + self.file.close() + + def __exit__(self, exc_type, exc_value, traceback): + os.remove(self.path) + + +def runtests(options): + os.environ['DJANGO_SETTINGS_MODULE'] = options.settings + + conf = load_settings(options.settings) + + if conf is None: + sys.stderr.write('Cannot load settings module: %s\n' % options.settings) + return sys.exit(1) + + settings.configure(**conf) + + redis_conf_path = options.conf or join(dirname(__file__), 'tests', 'redis.conf') + + server.configure(options.server_path, redis_conf_path, 0) + + try: + redis_conf_template = open(join(dirname(__file__), 'tests' ,'redis.conf.tpl')).read() + except OSError, IOError: + sys.stderr.write('Cannot find template for redis.conf.\n') + context = Context({ + 'redis_socket': join(dirname(abspath(__file__)), 'tests', 'redis.sock') + }) + + contents = Template(redis_conf_template).render(context) + + with TmpFile(redis_conf_path, contents): + with server: + from django.test.simple import DjangoTestSuiteRunner + runner = DjangoTestSuiteRunner(verbosity=options.verbosity, interactive=True, failfast=False) + failures = runner.run_tests(['testapp']) + + sys.exit(failures) + + +if __name__ == '__main__': + parser = OptionParser() + parser.add_option("-s", "--server", dest="server_path", action="store", + type="string", default=None, help="Path to the redis server executable") + parser.add_option("-c", "--conf", dest="conf", default=None, + help="Path to the redis configuration file.") + parser.add_option("-v", "--verbosity", dest="verbosity", default=1, type="int", + help="Change the verbostiy of the redis-server.") + parser.add_option("--settings", dest="settings", default="tests.settings", + help="Django settings module to use for the tests.") + + (options, args) = parser.parse_args() + + parent = dirname(abspath(__file__)) + sys.path.insert(0, parent) + + runtests(options) diff --git a/sockettests.py b/sockettests.py deleted file mode 100755 index 57479456..00000000 --- a/sockettests.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import sys -from os.path import dirname, abspath -from django.conf import settings - - -cache_settings = { - 'DATABASES': { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } - }, - 'INSTALLED_APPS': [ - 'tests.testapp', - ], - 'ROOT_URLCONF': 'tests.urls', - 'CACHES': { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '/tmp/redis.sock', - 'OPTIONS': { - 'DB': 15, - 'PASSWORD': 'yadayada', - 'PARSER_CLASS': 'redis.connection.HiredisParser' - }, - }, - }, -} - -if not settings.configured: - settings.configure(**cache_settings) - -from django.test.simple import DjangoTestSuiteRunner - -def runtests(*test_args): - if not test_args: - test_args = ['testapp'] - parent = dirname(abspath(__file__)) - sys.path.insert(0, parent) - runner = DjangoTestSuiteRunner(verbosity=1, interactive=True, failfast=False) - failures = runner.run_tests(test_args) - sys.exit(failures) - -if __name__ == '__main__': - runtests(*sys.argv[1:]) diff --git a/tcptests.py b/tcptests.py deleted file mode 100755 index 74234725..00000000 --- a/tcptests.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -import sys -from os.path import dirname, abspath -from django.conf import settings - - -cache_settings = { - 'DATABASES': { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } - }, - 'INSTALLED_APPS': [ - 'tests.testapp', - ], - 'ROOT_URLCONF': 'tests.urls', - 'CACHES': { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '127.0.0.1:6379', - 'OPTIONS': { - 'DB': 15, - 'PASSWORD': 'yadayada', - 'PARSER_CLASS': 'redis.connection.HiredisParser' - }, - }, - }, -} - - -if not settings.configured: - settings.configure(**cache_settings) - -from django.test.simple import DjangoTestSuiteRunner - -def runtests(*test_args): - if not test_args: - test_args = ['testapp'] - parent = dirname(abspath(__file__)) - sys.path.insert(0, parent) - runner = DjangoTestSuiteRunner(verbosity=1, interactive=True, failfast=False) - failures = runner.run_tests(test_args) - sys.exit(failures) - -if __name__ == '__main__': - runtests(*sys.argv[1:]) diff --git a/tests/base_settings.py b/tests/base_settings.py new file mode 100644 index 00000000..3ca25ad0 --- /dev/null +++ b/tests/base_settings.py @@ -0,0 +1,10 @@ +DEBUG = True +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + } +} +INSTALLED_APPS = [ + 'tests.testapp', +] +ROOT_URLCONF = 'tests.urls' diff --git a/tests/settings.py b/tests/hiredis_parser_settings.py similarity index 56% rename from tests/settings.py rename to tests/hiredis_parser_settings.py index 22c26f94..002c7d73 100644 --- a/tests/settings.py +++ b/tests/hiredis_parser_settings.py @@ -1,14 +1,5 @@ -DEBUG = True +from .base_settings import * -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - } -} - -INSTALLED_APPS = [ - 'tests.testapp', -] CACHES = { 'default': { @@ -17,8 +8,7 @@ 'OPTIONS': { # optional 'DB': 15, 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.HiredisParser', }, }, } - -ROOT_URLCONF = 'tests.urls' diff --git a/tests/python_parser_settings.py b/tests/python_parser_settings.py new file mode 100644 index 00000000..98de7e08 --- /dev/null +++ b/tests/python_parser_settings.py @@ -0,0 +1,14 @@ +from .base_settings import * + + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': '127.0.0.1:6379', + 'OPTIONS': { + 'DB': 15, + 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.PythonParser' + }, + }, +} diff --git a/tests/redis.conf.tpl b/tests/redis.conf.tpl new file mode 100644 index 00000000..2a631579 --- /dev/null +++ b/tests/redis.conf.tpl @@ -0,0 +1,525 @@ +# Redis configuration file example + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /var/run/redis.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# If you want you can bind a single interface, if the bind option is not +# specified all the interfaces will listen for incoming connections. +# +# bind 127.0.0.1 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +unixsocket {{ redis_socket }} +#unixsocketperm 755 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# Set server verbosity to 'debug' +# it can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also 'stdout' can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile stdout + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################# +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving at all commenting all the "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in an hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# distater will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usually even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since verison 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# Also the Append Only File will be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir ./ + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. Note that the configuration is local to the slave +# so for example it is possible to configure the slave to save the DB with a +# different interval, or to listen to another port, and so on. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave lost the connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extend you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets a timeout for both Bulk transfer I/O timeout and +# master data or ping response timeout. The default value is 60 seconds. +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# of hard to guess so that it will be still available for internal-use +# tools but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command renaming it into +# an empty string: +# +# rename-command CONFIG "" + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able ot configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# accordingly to the eviction policy selected (see maxmemmory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# an hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached? You can select among five behavior: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key accordingly to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with all the kind of policies, Redis will return an error on write +# operations, when there are not suitable keys for eviction. +# +# At the date of writing this commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") +# appendfilename appendonly.aof + +# The fsync() call tells the Operating System to actually write data on disk +# instead to wait for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log . Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec" that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving the durability of Redis is +# the same as "appendfsync none", that in practical terms means that it is +# possible to lost up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size will growth by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (or if no rewrite happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceed the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write commands was +# already issue by the script but the user don't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happens to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into an hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# active rehashing the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply form time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients +# slave -> slave clients and MONITOR clients +# pubsub -> clients subcribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled just setting it to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis server but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# include /path/to/local.conf +# include /path/to/other.conf diff --git a/tests/sockets_settings.py b/tests/sockets_settings.py new file mode 100644 index 00000000..855800bf --- /dev/null +++ b/tests/sockets_settings.py @@ -0,0 +1,16 @@ +from .base_settings import * +from os.path import join, dirname + + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': join(dirname(__file__), 'redis.sock'), + 'OPTIONS': { # optional + 'DB': 15, + 'PASSWORD': 'yadayada', + 'PARSER_CLASS': 'redis.connection.HiredisParser', + }, + }, +} + diff --git a/travis_run_all_tests.sh b/travis_run_all_tests.sh new file mode 100755 index 00000000..59e82551 --- /dev/null +++ b/travis_run_all_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./run_tests.py --settings=tests.settings -s ../redis/src/redis-server +./run_tests.py --settings=tests.python_parser_settings -s ../redis/src/redis-server +./run_tests.py --settings=tests.sockets_settings -s ../redis/src/redis-server \ No newline at end of file From f8d26fa77590d7389c1227ce3103a938dd18333e Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 15:24:06 -0700 Subject: [PATCH 02/15] Fixing .travis.yml to use the correct script. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 89b1afef..224bf399 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ branches: - experimental -script: travis_run_all_tests \ No newline at end of file +script: travis_run_all_tests.sh \ No newline at end of file From ac1e4fcf95bc4f54b63fa307e9d709fb11333d50 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 18:21:46 -0700 Subject: [PATCH 03/15] Removing flag to quiet the pip installs so I can debug better with travis. --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 224bf399..ba50ec2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,11 @@ env: #- DJANGO_VERSION=1.4 REDIS_VERSION=2.4 - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 install: - - pip install -q Django==$DJANGO_VERSION - - pip install -q -e git://github.com/sebleier/redis-py.git#egg=redis-py + - pip install Django==$DJANGO_VERSION + - pip install -e git://github.com/sebleier/redis-py.git#egg=redis-py - install_redis.sh branches: only: - master - experimental - - script: travis_run_all_tests.sh \ No newline at end of file From 51b93fb01ab60c211f2f51c3be10b55eb40a6b1d Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 22:29:15 -0700 Subject: [PATCH 04/15] Updating the .travis.yml file. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba50ec2f..a65e738e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,9 @@ env: install: - pip install Django==$DJANGO_VERSION - pip install -e git://github.com/sebleier/redis-py.git#egg=redis-py - - install_redis.sh + - ./install_redis.sh branches: only: - master - experimental -script: travis_run_all_tests.sh \ No newline at end of file +script: ./travis_run_all_tests.sh \ No newline at end of file From 6056c03bd015d167a662542afde25e612bf740a5 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 22:43:02 -0700 Subject: [PATCH 05/15] Adding 2.5 compat for with statement and adding hiredis to travis install yaml. --- .travis.yml | 2 +- run_tests.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a65e738e..ff99a4f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: #- DJANGO_VERSION=1.4 REDIS_VERSION=2.4 - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 install: - - pip install Django==$DJANGO_VERSION + - pip install hiredis Django==$DJANGO_VERSION - pip install -e git://github.com/sebleier/redis-py.git#egg=redis-py - ./install_redis.sh branches: diff --git a/run_tests.py b/run_tests.py index 9a435b6c..3b43fceb 100755 --- a/run_tests.py +++ b/run_tests.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import with_statement from optparse import OptionParser import os from os.path import dirname, abspath, join From 81f598c1c3033766d6fd8751893327a92091c9a9 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 22:54:46 -0700 Subject: [PATCH 06/15] Splitting up the pip install lines. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff99a4f9..9b00d8fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,9 @@ env: #- DJANGO_VERSION=1.4 REDIS_VERSION=2.4 - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 install: - - pip install hiredis Django==$DJANGO_VERSION + - pip install Django==$DJANGO_VERSION - pip install -e git://github.com/sebleier/redis-py.git#egg=redis-py + - pip install hiredis - ./install_redis.sh branches: only: From c2148baaf126c87b0a2375f98265d056e1c50ab9 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 23:27:48 -0700 Subject: [PATCH 07/15] Removing python 2.5 from the build. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9b00d8fc..37fea15a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.5" - "2.6" - "2.7" env: From 7abe947e8ba18790db2b2343affce979bbc6cc5c Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Thu, 24 May 2012 23:57:54 -0700 Subject: [PATCH 08/15] Changing verbosity to 1 for the redis server. --- run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 3b43fceb..791d6069 100755 --- a/run_tests.py +++ b/run_tests.py @@ -50,7 +50,7 @@ def runtests(options): redis_conf_path = options.conf or join(dirname(__file__), 'tests', 'redis.conf') - server.configure(options.server_path, redis_conf_path, 0) + server.configure(options.server_path, redis_conf_path, 1) try: redis_conf_template = open(join(dirname(__file__), 'tests' ,'redis.conf.tpl')).read() From 80b1d776779646f3568022e27371c86e48fe76d2 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Fri, 25 May 2012 00:07:24 -0700 Subject: [PATCH 09/15] Changing the port for the redis build. --- tests/redis.conf.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/redis.conf.tpl b/tests/redis.conf.tpl index 2a631579..764c2a4e 100644 --- a/tests/redis.conf.tpl +++ b/tests/redis.conf.tpl @@ -22,7 +22,7 @@ pidfile /var/run/redis.pid # Accept connections on the specified port, default is 6379. # If port 0 is specified Redis will not listen on a TCP socket. -port 6379 +port 6380 # If you want you can bind a single interface, if the bind option is not # specified all the interfaces will listen for incoming connections. From a34cc6e9c4b894f0d7bdd49aeb9856719f851b45 Mon Sep 17 00:00:00 2001 From: Sean Bleier Date: Fri, 25 May 2012 00:12:18 -0700 Subject: [PATCH 10/15] Changing verbosity back to 0 and adding more configurations. --- .travis.yml | 16 ++++++++-------- run_tests.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37fea15a..772f7edf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,14 @@ python: - "2.6" - "2.7" env: - #- DJANGO_VERSION=1.2 REDIS_VERSION=2.2 - #- DJANGO_VERSION=1.2 REDIS_VERSION=2.4 - #- DJANGO_VERSION=1.2 REDIS_VERSION=2.6 - #- DJANGO_VERSION=1.3 REDIS_VERSION=2.2 - #- DJANGO_VERSION=1.3 REDIS_VERSION=2.4 - #- DJANGO_VERSION=1.3 REDIS_VERSION=2.6 - #- DJANGO_VERSION=1.4 REDIS_VERSION=2.2 - #- DJANGO_VERSION=1.4 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.2 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.2 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.2 REDIS_VERSION=2.6 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.4 + - DJANGO_VERSION=1.3 REDIS_VERSION=2.6 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.2 + - DJANGO_VERSION=1.4 REDIS_VERSION=2.4 - DJANGO_VERSION=1.4 REDIS_VERSION=2.6 install: - pip install Django==$DJANGO_VERSION diff --git a/run_tests.py b/run_tests.py index 791d6069..3b43fceb 100755 --- a/run_tests.py +++ b/run_tests.py @@ -50,7 +50,7 @@ def runtests(options): redis_conf_path = options.conf or join(dirname(__file__), 'tests', 'redis.conf') - server.configure(options.server_path, redis_conf_path, 1) + server.configure(options.server_path, redis_conf_path, 0) try: redis_conf_template = open(join(dirname(__file__), 'tests' ,'redis.conf.tpl')).read() From f8111598726b424b1850b4a18e14d96b295d11d3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Jun 2012 19:01:23 +0530 Subject: [PATCH 11/15] Adding hset, hget, hset_many, hget_many, hincr implementation --- redis_cache/cache.py | 120 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/redis_cache/cache.py b/redis_cache/cache.py index 055073d6..8f490621 100644 --- a/redis_cache/cache.py +++ b/redis_cache/cache.py @@ -188,6 +188,23 @@ def get(self, key, default=None, version=None): result = self.unpickle(value) return result + def hget(self, name, key, default=None, version=None): + """ + Retrieve a value from the cache as hash. + + Returns unpickled value if key is found, the default if not. + """ + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + value = self._client.hget(name, key) + if value is None: + return default + try: + result = int(value) + except (ValueError, TypeError): + result = self.unpickle(value) + return result + def _set(self, key, value, timeout, client): if timeout == 0: return client.set(key, value) @@ -196,6 +213,15 @@ def _set(self, key, value, timeout, client): else: return False + def _hset(self, name, key, value, timeout, client): + """Set value in hash""" + # Redis call + status = client.hset( name, key, value) + if timeout > 0: + # Set expire time + client.expire(name, timeout) + return status + def set(self, key, value, timeout=None, version=None, client=None): """ Persist a value to the cache, and set an optional expiration time. @@ -217,6 +243,30 @@ def set(self, key, value, timeout=None, version=None, client=None): # result is a boolean return result + def hset(self, outerk, hashk, value, timeout=None, version=None, client=None): + """ + Sets field in the hash stored at key to value + and set an optional expiration time. + """ + if not client: + client = self._client + outerk = self.make_key(outerk, version=version) + hashk = self.make_key(hashk, version=version) + if timeout is None: + # To store it persistently + timeout = 0 + try: + value = float(value) + # If you lose precision from the typecast to str, then pickle value + if int(value) != value: + raise TypeError + except (ValueError, TypeError): + result = self._hset(outerk, hashk, pickle.dumps(value), int(timeout), client) + else: + result = self._hset(outerk, hashk, int(value), int(timeout), client) + # result is a boolean + return result + def delete(self, key, version=None): """ Remove a key from the cache. @@ -267,6 +317,29 @@ def get_many(self, keys, version=None): recovered_data[map_keys[key]] = value return recovered_data + def hget_many(self, name, keys, version=None): + """ + Retrieve many keys from hash. + Keys is the list of the key in the hash. + Result is returned as key value pair dict. + """ + recovered_data = SortedDict() + new_keys = map(lambda key: self.make_key(key, version=version), keys) + # Redis call + results = self._client.hmget(name, keys) + # Iterate to convert the result into proper format. + for key, value in zip(new_keys, results): + if value is None: + continue + try: + value = int(value) + except (ValueError, TypeError): + value = self.unpickle(value) + if isinstance(value, basestring): + value = smart_unicode(value) + recovered_data[key] = value + return recovered_data + def set_many(self, data, timeout=None, version=None): """ Set a bunch of values in the cache at once from a dict of key/value @@ -280,6 +353,35 @@ def set_many(self, data, timeout=None, version=None): self.set(key, value, timeout, version=version, client=pipeline) pipeline.execute() + + def hset_many(self, name, mapping_dict, timeout=None, version=None): + """ + Set a bunch of values in the cache at once from a dict of key/value + pairs. This is much more efficient than calling hset() multiple times. + + If timeout is given, that timeout will be used for the key; otherwise + stored persistently + """ + # Iterate to convert the values to appropriate format. + for key, value in mapping_dict.items(): + try: + value = float(value) + # If you lose precision from the typecast to str, then pickle value + if int(value) != value: + raise TypeError + except (ValueError, TypeError): + value = pickle.dumps(value) + else: + value = int(value) + mapping_dict[key] = value + + # Redis call + result = self._client.hmset(name, mapping_dict) + if timeout is not None: + # Set timeout of the name + self._client.expire(name, timeout) + return result + def incr(self, key, delta=1, version=None): """ Add delta to value in the cache. If the key does not exist, raise a @@ -296,6 +398,24 @@ def incr(self, key, delta=1, version=None): self.set(key, value) return value + def hincr(self, name, key, delta=1, version=None): + """ + Add delta to value in the cache of the hash. + If the key does not exist, raise a + ValueError exception. + """ + key = self.make_key(key, version=version) + exists = self._client.exists(key) + if not exists: + raise ValueError("Key '%s' not found" % key) + try: + # Redis call + value = self._client.hincr(name, key, delta) + except redis.ResponseError: + value = self.hget(name, key) + 1 + # Redis call + self.hset(name, key, value) + return value class RedisCache(CacheClass): """ From cfbab36a6c72274b137cc85fb324d79cfb40f623 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Jun 2012 15:55:15 +0530 Subject: [PATCH 12/15] Fixing the bug in hset_many and adding has_hkey implementation --- redis_cache/cache.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/redis_cache/cache.py b/redis_cache/cache.py index 8f490621..1f4b8b2d 100644 --- a/redis_cache/cache.py +++ b/redis_cache/cache.py @@ -325,8 +325,10 @@ def hget_many(self, name, keys, version=None): """ recovered_data = SortedDict() new_keys = map(lambda key: self.make_key(key, version=version), keys) + map_keys = dict(zip(new_keys, keys)) + name = self.make_key(name, version=version) # Redis call - results = self._client.hmget(name, keys) + results = self._client.hmget(name, new_keys) # Iterate to convert the result into proper format. for key, value in zip(new_keys, results): if value is None: @@ -337,7 +339,7 @@ def hget_many(self, name, keys, version=None): value = self.unpickle(value) if isinstance(value, basestring): value = smart_unicode(value) - recovered_data[key] = value + recovered_data[map_keys[key]] = value return recovered_data def set_many(self, data, timeout=None, version=None): @@ -362,6 +364,9 @@ def hset_many(self, name, mapping_dict, timeout=None, version=None): If timeout is given, that timeout will be used for the key; otherwise stored persistently """ + # Convert the keys to version format + mapping_dict = dict( (self.make_key(key, version), value) for key, value in mapping_dict.items() ) + name = self.make_key(name, version=version) # Iterate to convert the values to appropriate format. for key, value in mapping_dict.items(): try: @@ -417,6 +422,14 @@ def hincr(self, name, key, delta=1, version=None): self.hset(name, key, value) return value + def has_hkey(self, name, key, version=None): + """ + The name and key exist in the hash + """ + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) + return self._client.hexists(name, key) + class RedisCache(CacheClass): """ A subclass that is supposed to be used on Django >= 1.3. From 532f6477c5e18cddfc26adb6697a538e73d093f5 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Jun 2012 15:58:16 +0530 Subject: [PATCH 13/15] Adding unit test for the new implemented hash methods --- tests/testapp/tests.py | 88 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/testapp/tests.py b/tests/testapp/tests.py index 69a188da..64bcbac3 100644 --- a/tests/testapp/tests.py +++ b/tests/testapp/tests.py @@ -311,7 +311,7 @@ def test_incr_version(self): self.assertEqual(self.cache.get(new_key), 'spam') def test_incr_with_pickled_integer(self): - "Testing case where there exists a pickled integer and we increment it" + #Testing case where there exists a pickled integer and we increment it number = 42 key = self.cache.make_key("key") @@ -356,6 +356,92 @@ def test_multiple_connection_pool_connections(self): c3 = get_cache('redis_cache.cache://127.0.0.1:6379?db=15') self.assertEqual(len(pool._connection_pools), 2) + def test_hset(self): + # Set in a hash + self.cache.hset('a', 'a1', 'A') + self.assertEqual(self.cache.hget('a','a1'),'A') + + def test_float_hcaching(self): + # Set float value in a hash + self.cache.hset('a', 'a1', 1.1) + a = self.cache.hget('a', 'a1') + self.assertEqual(a, 1.1) + + def test_string_float_hcaching(self): + self.cache.hset('a', 'a1', '1.1') + a = self.cache.hget('a','a1') + self.assertEqual(a, 1.1) + + def test_hget_many(self): + # Multiple cache keys can be returned using get_many + self.cache.hset('a', 'a1', 'A1') + self.cache.hset('a', 'a2', 'A2') + self.cache.hset('a', 'a3', 'A3') + self.assertEqual(self.cache.hget_many('a',['a1','a2']), {'a1': 'A1', 'a2': 'A2'}) + self.assertEqual(self.cache.hget_many('a',['a3']), {'a3': 'A3'}) + + def test_hset_many(self): + # Multiple cache keys can be set using hset_many in one call + self.cache.hset_many('a', {'a1':'A1', 'a2':'A2', 'a3':'A3'}) + self.assertEqual(self.cache.hget_many('a',['a1','a2']), {'a1': 'A1', 'a2': 'A2'}) + self.assertEqual(self.cache.hget('a','a1'), 'A1') + + def test_hexpiration(self): + # Cache values can be set to expire + self.cache.hset('expire1', 'key1', 1, 1) + self.cache.hset('expire2', 'key2', 1, 1) + self.cache.hset('expire3', 'key3', 1, 1) + + time.sleep(2) + self.assertEqual(self.cache.hget("expire1","key1"), None) + + self.cache.hset("expire2", "key2", "newvalue") + self.assertEqual(self.cache.hget("expire2", "key2"), "newvalue") + + def test_hset_expiration_timeout_None(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value) + self.assertTrue(self.cache._client.ttl(key) is None) + + def test_hset_expiration_timeout_zero(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value, timeout=0) + self.assertTrue(self.cache._client.ttl(key) is None) + self.assertTrue(self.cache.has_hkey(key,'a')) + + def test_hset_expiration_timeout_negative(self): + key, value = self.cache.make_key('key'), 'value' + self.cache.hset(key, 'a', value, timeout=-1) + self.assertTrue(self.cache._client.ttl(key) is None) + + def test_hash_unicode(self): + # Unicode values can be cached + stuff = { + u'ascii': (u'key1',u'ascii_value'), + u'unicode_ascii': (u'key2',u'Iñtërnâtiônàlizætiøn1'), + u'Iñtërnâtiônàlizætiøn': (u'key3',u'Iñtërnâtiônàlizætiøn2'), + u'ascii': (u'key4',{u'x' : 1 }) + } + for (key, value) in stuff.items(): + self.cache.hset(key, value[0], value[1]) + self.assertEqual(self.cache.hget(key, value[0]), value[1]) + + def test_hash_binary_string(self): + # Binary strings should be cachable + from zlib import compress, decompress + value = 'value_to_be_compressed' + compressed_value = compress(value) + self.cache.hset('binary1', 'b1', compressed_value) + compressed_result = self.cache.hget('binary1','b1') + self.assertEqual(compressed_value, compressed_result) + self.assertEqual(value, decompress(compressed_result)) + + def test_has_hkey(self): + # Presence of hash key + self.cache.hset('A','A1',100) + self.assertEqual(self.cache.has_hkey('A','A1'),True) + self.assertEqual(self.cache.has_hkey('B','B1'),False) + self.assertEqual(self.cache.has_hkey('A','A10'),False) if __name__ == '__main__': unittest.main() From fcab337ba6199ce9e277b54ce6b0547b5acddd8d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jun 2012 00:58:52 +0530 Subject: [PATCH 14/15] Adding benchmark methods for the hashes implementation --- tests/benchmark.py | 49 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/benchmark.py b/tests/benchmark.py index 8ee50f66..5af02f37 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -8,7 +8,6 @@ python benchmark.py master python benchamrk.py some-branch """ - import os import sys from time import time @@ -48,12 +47,16 @@ def tearDown(self): pass def timetrial(self): + self.cache = cache.get_cache('default') + # Initializing the redis to basic settings + self.cache._client.flushall() self.setUp() start = time() self.run() t = time() - start self.tearDown() - return t + final_usuage = self.cache._client.info()['used_memory_human'] + return t, final_usuage def run(self): pass @@ -63,7 +66,7 @@ def run_benchmarks(cls): for benchmark in cls.benchmarks: benchmark = benchmark() print benchmark.__doc__ - print "Time: %s" % (benchmark.timetrial()) + print "Time: %s; Memory: %s" % (benchmark.timetrial()) class GetAndSetBenchmark(Benchmark): @@ -96,6 +99,8 @@ def setUp(self): self.values[h(h(i))] = h(i) self.ints.append(i) self.strings.append(h(i)) + for k, v in self.values.items(): + self.cache.set(k, v) def run(self): for i in self.ints: @@ -116,6 +121,42 @@ def run(self): self.cache.set_many(self.values) value = self.cache.get_many(self.values.keys()) +class HGetAndHSetBenchmark(Benchmark): + "Settings and Getting Mixed for Hashes" + + def setUp(self): + self.cache = cache.get_cache('default') + self.values = {} + # 100*300 = 30000 + for i in range(100): + self.values[h(i)] = i + self.values[h(h(i))] = h(i) + + def run(self): + for k, v in self.values.items(): + for i in range(300): + self.cache.hset(k, str(i), v) + for k, v in self.values.items(): + for i in range(300): + self.cache.hget(k, str(i)) + +class HMsetAndHMGet(Benchmark): + "Getting and setting many mixed values for Hashes" + + def setUp(self): + self.cache = cache.get_cache('default') + self.values = {} + for i in range(100): + self.values[h(i)] = i + self.values[h(h(i))] = h(i) + + def run(self): + for k, v in self.values.items(): + mapping_dict = dict( (str(i), v) for i in range(300) ) + self.cache.hset_many(k, mapping_dict) + for k, v in self.values.items(): + many_keys = [ str(i) for i in range(300) ] + self.cache.hget_many(k, many_keys) if __name__ == "__main__": - Benchmark.run_benchmarks() \ No newline at end of file + Benchmark.run_benchmarks() From b69892093114048a62795c603031dd5dece01918 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jun 2012 09:04:25 +0530 Subject: [PATCH 15/15] Changing name of variable and fixing a bug in hincr --- redis_cache/cache.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/redis_cache/cache.py b/redis_cache/cache.py index 1f4b8b2d..01077247 100644 --- a/redis_cache/cache.py +++ b/redis_cache/cache.py @@ -243,15 +243,15 @@ def set(self, key, value, timeout=None, version=None, client=None): # result is a boolean return result - def hset(self, outerk, hashk, value, timeout=None, version=None, client=None): + def hset(self, name, key, value, timeout=None, version=None, client=None): """ Sets field in the hash stored at key to value and set an optional expiration time. """ if not client: client = self._client - outerk = self.make_key(outerk, version=version) - hashk = self.make_key(hashk, version=version) + name = self.make_key(name, version=version) + key = self.make_key(key, version=version) if timeout is None: # To store it persistently timeout = 0 @@ -261,9 +261,9 @@ def hset(self, outerk, hashk, value, timeout=None, version=None, client=None): if int(value) != value: raise TypeError except (ValueError, TypeError): - result = self._hset(outerk, hashk, pickle.dumps(value), int(timeout), client) + result = self._hset(name, key, pickle.dumps(value), int(timeout), client) else: - result = self._hset(outerk, hashk, int(value), int(timeout), client) + result = self._hset(name, key, int(value), int(timeout), client) # result is a boolean return result @@ -409,13 +409,14 @@ def hincr(self, name, key, delta=1, version=None): If the key does not exist, raise a ValueError exception. """ + name = self.make_key(name, version=version) key = self.make_key(key, version=version) - exists = self._client.exists(key) + exists = self._client.hexists(name, key) if not exists: raise ValueError("Key '%s' not found" % key) try: # Redis call - value = self._client.hincr(name, key, delta) + value = self._client.hincrby(name, key, delta) except redis.ResponseError: value = self.hget(name, key) + 1 # Redis call