Skip to content

Add configurable deletion strategy for Redis repository operations #2294 #3162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -103,6 +104,7 @@
* @author Mark Paluch
* @author Andrey Muchnik
* @author John Blum
* @author Kim Sumin
* @since 1.7
*/
public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
@@ -126,6 +128,7 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
private EnableKeyspaceEvents enableKeyspaceEvents = EnableKeyspaceEvents.OFF;
private @Nullable String keyspaceNotificationsConfigParameter = null;
private ShadowCopy shadowCopy = ShadowCopy.DEFAULT;
private DeletionStrategy deletionStrategy = DeletionStrategy.DEL;

/**
* Lifecycle state of this factory.
@@ -134,6 +137,43 @@ enum State {
CREATED, STARTING, STARTED, STOPPING, STOPPED, DESTROYED;
}

/**
* Strategy for deleting Redis keys in Repository operations.
* <p>
* Allows configuration of whether to use synchronous {@literal DEL} or asynchronous {@literal UNLINK} commands for
* key deletion operations.
*
* @author [Your Name]
* @since 3.6
* @see <a href="https://redis.io/commands/del">Redis DEL</a>
* @see <a href="https://redis.io/commands/unlink">Redis UNLINK</a>
*/
public enum DeletionStrategy {

/**
* Use Redis {@literal DEL} command for key deletion.
* <p>
* 기key from memory. The command blocks until the key is completely removed, which can cause performance issues when
* deleting large data structures under high load.
* <p>
* This is the default strategy for backward compatibility.
*/
DEL,

/**
* Use Redis {@literal UNLINK} command for key deletion.
* <p>
* This is a non-blocking operation that asynchronously removes the key. The key is immediately removed from the
* keyspace, but the actual memory reclamation happens in the background, providing better performance for
* applications with frequent updates on existing keys.
* <p>
* Requires Redis 4.0 or later.
*
* @since Redis 4.0
*/
UNLINK
}

/**
* Creates new {@link RedisKeyValueAdapter} with default {@link RedisMappingContext} and default
* {@link RedisCustomConversions}.
@@ -228,7 +268,7 @@ public Object put(Object id, Object item, String keyspace) {
byte[] key = toBytes(rdo.getId());
byte[] objectKey = createKey(rdo.getKeyspace(), rdo.getId());

boolean isNew = connection.del(objectKey) == 0;
boolean isNew = applyDeletionStrategy(connection, objectKey) == 0;

connection.hMSet(objectKey, rdo.getBucket().rawMap());

@@ -245,11 +285,11 @@ public Object put(Object id, Object item, String keyspace) {
byte[] phantomKey = ByteUtils.concat(objectKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);

if (expires(rdo)) {
connection.del(phantomKey);
applyDeletionStrategy(connection, phantomKey);
connection.hMSet(phantomKey, rdo.getBucket().rawMap());
connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
} else if (!isNew) {
connection.del(phantomKey);
applyDeletionStrategy(connection, phantomKey);
}
}

@@ -323,7 +363,7 @@ public <T> T delete(Object id, String keyspace, Class<T> type) {

redisOps.execute((RedisCallback<Void>) connection -> {

connection.del(keyToDelete);
applyDeletionStrategy(connection, keyToDelete);
connection.sRem(binKeyspace, binId);
new IndexWriter(connection, converter).removeKeyFromIndexes(keyspace, binId);

@@ -335,7 +375,7 @@ public <T> T delete(Object id, String keyspace, Class<T> type) {

byte[] phantomKey = ByteUtils.concat(keyToDelete, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);

connection.del(phantomKey);
applyDeletionStrategy(connection, phantomKey);
}
}
return null;
@@ -485,7 +525,7 @@ public void update(PartialUpdate<?> update) {
connection.persist(redisKey);

if (keepShadowCopy()) {
connection.del(ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX));
applyDeletionStrategy(connection, ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX));
}
}
}
@@ -495,6 +535,18 @@ public void update(PartialUpdate<?> update) {
});
}

/**
* Apply the configured deletion strategy to delete the given key.
*
* @param connection the Redis connection
* @param key the key to delete
* @return the number of keys that were removed
*/
private Long applyDeletionStrategy(RedisConnection connection, byte[] key) {
return Objects
.requireNonNull(deletionStrategy == DeletionStrategy.UNLINK ? connection.unlink(key) : connection.del(key));
}

private RedisUpdateObject fetchDeletePathsFromHashAndUpdateIndex(RedisUpdateObject redisUpdateObject, String path,
RedisConnection connection) {

@@ -704,6 +756,30 @@ public boolean isRunning() {
return State.STARTED.equals(this.state.get());
}

/**
* Configure the deletion strategy for Redis keys.
* <p>
* {@link DeletionStrategy#DEL DEL} performs synchronous key deletion, while {@link DeletionStrategy#UNLINK UNLINK}
* performs asynchronous deletion which can improve performance under high load scenarios.
*
* @param deletionStrategy the strategy to use for key deletion operations
* @since 3.6
*/
public void setDeletionStrategy(DeletionStrategy deletionStrategy) {
Assert.notNull(deletionStrategy, "DeletionStrategy must not be null");
this.deletionStrategy = deletionStrategy;
}

/**
* Get the current deletion strategy.
*
* @return the current deletion strategy
* @since 3.6
*/
public DeletionStrategy getDeletionStrategy() {
return this.deletionStrategy;
}

/**
* @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
* @since 1.8
@@ -792,7 +868,7 @@ private void initKeyExpirationListener(RedisMessageListenerContainer messageList

if (this.expirationListener.get() == null) {
MappingExpirationListener listener = new MappingExpirationListener(messageListenerContainer, this.redisOps,
this.converter, this.shadowCopy);
this.converter, this.shadowCopy, this.deletionStrategy);

listener.setKeyspaceNotificationsConfigParameter(keyspaceNotificationsConfigParameter);

@@ -819,17 +895,19 @@ static class MappingExpirationListener extends KeyExpirationEventMessageListener
private final RedisOperations<?, ?> ops;
private final RedisConverter converter;
private final ShadowCopy shadowCopy;
private final DeletionStrategy deletionStrategy;

/**
* Creates new {@link MappingExpirationListener}.
*/
MappingExpirationListener(RedisMessageListenerContainer listenerContainer, RedisOperations<?, ?> ops,
RedisConverter converter, ShadowCopy shadowCopy) {
RedisConverter converter, ShadowCopy shadowCopy, DeletionStrategy deletionStrategy) {

super(listenerContainer);
this.ops = ops;
this.converter = converter;
this.shadowCopy = shadowCopy;
this.deletionStrategy = deletionStrategy;
}

@Override
@@ -883,7 +961,11 @@ private Object readShadowCopy(byte[] key) {
Map<byte[], byte[]> phantomValue = connection.hGetAll(phantomKey);

if (!CollectionUtils.isEmpty(phantomValue)) {
connection.del(phantomKey);
if (deletionStrategy == DeletionStrategy.UNLINK) {
connection.unlink(phantomKey);
} else {
connection.del(phantomKey);
}
}

return phantomValue;
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.springframework.context.annotation.Import;
import org.springframework.data.keyvalue.core.KeyValueOperations;
import org.springframework.data.keyvalue.repository.config.QueryCreatorType;
import org.springframework.data.redis.core.RedisKeyValueAdapter.DeletionStrategy;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
import org.springframework.data.redis.core.RedisOperations;
@@ -47,6 +48,7 @@
*
* @author Christoph Strobl
* @author Mark Paluch
* @author Kim Sumin
* @since 1.7
*/
@Target(ElementType.TYPE)
@@ -129,7 +131,9 @@

/**
* Configure a specific {@link BeanNameGenerator} to be used when creating the repositoy beans.
* @return the {@link BeanNameGenerator} to be used or the base {@link BeanNameGenerator} interface to indicate context default.
*
* @return the {@link BeanNameGenerator} to be used or the base {@link BeanNameGenerator} interface to indicate
* context default.
* @since 3.4
*/
Class<? extends BeanNameGenerator> nameGenerator() default BeanNameGenerator.class;
@@ -204,4 +208,20 @@
*/
String keyspaceNotificationsConfigParameter() default "Ex";

/**
* Configure the deletion strategy for Redis keys during repository operations.
* <p>
* {@link DeletionStrategy#DEL DEL} uses synchronous deletion (blocking), while {@link DeletionStrategy#UNLINK UNLINK}
* uses asynchronous deletion (non-blocking).
* <p>
* {@literal UNLINK} can provide better performance for applications with frequent updates on existing keys,
* especially when dealing with large data structures under high load.
* <p>
* Requires Redis 4.0 or later when using {@link DeletionStrategy#UNLINK}.
*
* @return the deletion strategy to use
* @since 3.6
* @see DeletionStrategy
*/
DeletionStrategy deletionStrategy() default DeletionStrategy.DEL;
}
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.springframework.data.keyvalue.repository.config.KeyValueRepositoryConfigurationExtension;
import org.springframework.data.redis.core.RedisHash;
import org.springframework.data.redis.core.RedisKeyValueAdapter;
import org.springframework.data.redis.core.RedisKeyValueAdapter.DeletionStrategy;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
import org.springframework.data.redis.core.RedisKeyValueTemplate;
@@ -44,6 +45,7 @@
*
* @author Christoph Strobl
* @author Mark Paluch
* @author Kim Sumin
* @since 1.7
*/
public class RedisRepositoryConfigurationExtension extends KeyValueRepositoryConfigurationExtension {
@@ -145,7 +147,9 @@ private static AbstractBeanDefinition createRedisKeyValueAdapter(RepositoryConfi
configuration.getRequiredAttribute("enableKeyspaceEvents", EnableKeyspaceEvents.class)) //
.addPropertyValue("keyspaceNotificationsConfigParameter",
configuration.getAttribute("keyspaceNotificationsConfigParameter", String.class).orElse("")) //
.addPropertyValue("shadowCopy", configuration.getRequiredAttribute("shadowCopy", ShadowCopy.class));
.addPropertyValue("shadowCopy", configuration.getRequiredAttribute("shadowCopy", ShadowCopy.class))
.addPropertyValue("deletionStrategy",
configuration.getRequiredAttribute("deletionStrategy", DeletionStrategy.class));

configuration.getAttribute("messageListenerContainerRef")
.ifPresent(it -> builder.addPropertyReference("messageListenerContainer", it));
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@
/**
* @author Lucian Torje
* @author Christoph Strobl
* @author Kim Sumin
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
@@ -58,7 +59,7 @@ void testOnNonKeyExpiration() {
byte[] key = "testKey".getBytes();
when(message.getBody()).thenReturn(key);
listener = new RedisKeyValueAdapter.MappingExpirationListener(listenerContainer, redisOperations, redisConverter,
RedisKeyValueAdapter.ShadowCopy.ON);
RedisKeyValueAdapter.ShadowCopy.ON, RedisKeyValueAdapter.DeletionStrategy.DEL);

listener.onMessage(message, null);

@@ -74,7 +75,7 @@ void testOnValidKeyExpirationWithShadowCopiesDisabled() {
when(message.getBody()).thenReturn(key);

listener = new RedisKeyValueAdapter.MappingExpirationListener(listenerContainer, redisOperations, redisConverter,
RedisKeyValueAdapter.ShadowCopy.OFF);
RedisKeyValueAdapter.ShadowCopy.OFF, RedisKeyValueAdapter.DeletionStrategy.DEL);
listener.setApplicationEventPublisher(eventList::add);
listener.onMessage(message, null);

@@ -97,7 +98,7 @@ void testOnValidKeyExpirationWithShadowCopiesEnabled() {
when(conversionService.convert(any(), eq(byte[].class))).thenReturn("foo".getBytes());

listener = new RedisKeyValueAdapter.MappingExpirationListener(listenerContainer, redisOperations, redisConverter,
RedisKeyValueAdapter.ShadowCopy.ON);
RedisKeyValueAdapter.ShadowCopy.ON, RedisKeyValueAdapter.DeletionStrategy.DEL);
listener.setApplicationEventPublisher(eventList::add);
listener.onMessage(message, null);

Original file line number Diff line number Diff line change
@@ -56,6 +56,7 @@
* @author Christoph Strobl
* @author Mark Paluch
* @author Andrey Muchnik
* @author Kim Sumin
*/
@ExtendWith(LettuceConnectionFactoryExtension.class)
public class RedisKeyValueAdapterTests {
@@ -788,6 +789,61 @@ void updateWithRefreshTtlAndWithoutPositiveTtlShouldDeletePhantomKey() {
assertThat(template.hasKey("persons:1:phantom")).isFalse();
}

@Test // GH-2294
void shouldUseDELByDefault() {
// given
RedisKeyValueAdapter adapter = new RedisKeyValueAdapter(template, mappingContext);

// when & then
assertThat(adapter.getDeletionStrategy()).isEqualTo(RedisKeyValueAdapter.DeletionStrategy.DEL);
}

@Test // GH -2294
void shouldAllowUNLINKConfiguration() {
// given
RedisKeyValueAdapter adapter = new RedisKeyValueAdapter(template, mappingContext);

// when
adapter.setDeletionStrategy(RedisKeyValueAdapter.DeletionStrategy.UNLINK);

// then
assertThat(adapter.getDeletionStrategy()).isEqualTo(RedisKeyValueAdapter.DeletionStrategy.UNLINK);
}

@Test // GH-2294
void shouldRejectNullDeletionStrategy() {
// given
RedisKeyValueAdapter adapter = new RedisKeyValueAdapter(template, mappingContext);

// when & then
assertThatIllegalArgumentException().isThrownBy(() -> adapter.setDeletionStrategy(null))
.withMessageContaining("DeletionStrategy must not be null");
}

@Test // GH-2294
void shouldMaintainFunctionalityWithUNLINKStrategy() {
// given
adapter.setDeletionStrategy(RedisKeyValueAdapter.DeletionStrategy.UNLINK);

Person person = new Person();
person.id = "unlink-test";
person.firstname = "test";

// when & then
adapter.put(person.id, person, "persons");
assertThat(adapter.get(person.id, "persons", Person.class)).isNotNull();

person.firstname = "updated";
adapter.put(person.id, person, "persons");

Person result = adapter.get(person.id, "persons", Person.class);
assertThat(result.firstname).isEqualTo("updated");

adapter.delete(person.id, "persons");
assertThat(adapter.get(person.id, "persons", Person.class)).isNull();
}


/**
* Wait up to 5 seconds until {@code key} is no longer available in Redis.
*
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@
import static org.assertj.core.api.Assertions.*;

import java.util.Collection;
import java.util.Objects;

import org.junit.jupiter.api.Test;

@@ -32,6 +33,7 @@
import org.springframework.data.annotation.Id;
import org.springframework.data.keyvalue.repository.KeyValueRepository;
import org.springframework.data.redis.core.RedisHash;
import org.springframework.data.redis.core.RedisKeyValueAdapter.DeletionStrategy;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.repository.Repository;
import org.springframework.data.repository.config.AnnotationRepositoryConfigurationSource;
@@ -43,6 +45,7 @@
*
* @author Christoph Strobl
* @author Mark Paluch
* @author Kim Sumin
*/
class RedisRepositoryConfigurationExtensionUnitTests {

@@ -117,6 +120,22 @@ void explicitlyEmptyKeyspaceNotificationsConfigParameterShouldBeCapturedCorrectl
assertThat(getKeyspaceNotificationsConfigParameter(beanDefintionRegistry)).isEqualTo("");
}

@Test // GH-2294
void picksUpDeletionStrategyDefaultCorrectly() {
metadata = new StandardAnnotationMetadata(ConfigWithDefaultDeletionStrategy.class, true);
BeanDefinitionRegistry beanDefinitionRegistry = getBeanDefinitionRegistry();

assertThat(getDeletionStrategy(beanDefinitionRegistry)).isEqualTo(DeletionStrategy.DEL);
}

@Test // GH-2294
void picksUpDeletionStrategyUnlinkCorrectly() {
metadata = new StandardAnnotationMetadata(ConfigWithUnlinkDeletionStrategy.class, true);
BeanDefinitionRegistry beanDefinitionRegistry = getBeanDefinitionRegistry();

assertThat(getDeletionStrategy(beanDefinitionRegistry)).isEqualTo(DeletionStrategy.UNLINK);
}

private static void assertDoesNotHaveRepo(Class<?> repositoryInterface,
Collection<RepositoryConfiguration<RepositoryConfigurationSource>> configs) {

@@ -166,6 +185,11 @@ private Object getKeyspaceNotificationsConfigParameter(BeanDefinitionRegistry be
.getPropertyValue("keyspaceNotificationsConfigParameter").getValue();
}

private Object getDeletionStrategy(BeanDefinitionRegistry beanDefinitionRegistry) {
return Objects.requireNonNull(beanDefinitionRegistry.getBeanDefinition("redisKeyValueAdapter").getPropertyValues()
.getPropertyValue("deletionStrategy").getValue());
}

@EnableRedisRepositories(considerNestedRepositories = true, enableKeyspaceEvents = EnableKeyspaceEvents.ON_STARTUP)
private static class Config {

@@ -188,6 +212,12 @@ private static class ConfigWithEmptyConfigParameter {

}

@EnableRedisRepositories(considerNestedRepositories = true)
private static class ConfigWithDefaultDeletionStrategy {}

@EnableRedisRepositories(considerNestedRepositories = true, deletionStrategy = DeletionStrategy.UNLINK)
private static class ConfigWithUnlinkDeletionStrategy {}

@RedisHash
static class Sample {
@Id String id;