RabbitMQ Cluster 常见错误码原因与解决办法

2017-09-26 本文已影响333人野餐先生

1.场景

1.1 404 NOT_FOUND

org.springframework.amqp.AmqpIOException: java.io.IOException
    at org.springframework.amqp.rabbit.connection.RabbitUtils.convertRabbitAccessException(RabbitUtils.java:109) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.RabbitAccessor.convertRabbitAccessException(RabbitAccessor.java:106) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:413) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitAdmin.initialize(RabbitAdmin.java:298) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitAdmin$10.onCreate(RabbitAdmin.java:232) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.CompositeConnectionListener.onCreate(CompositeConnectionListener.java:31) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createConnection(CachingConnectionFactory.java:181) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createBareChannel(CachingConnectionFactory.java:171) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.access$200(CachingConnectionFactory.java:48) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.connection.CachingConnectionFactory$CachedChannelInvocationHandler.invoke(CachingConnectionFactory.java:296) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at com.sun.proxy.$Proxy35.basicPublish(Unknown Source) ~[na:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.doSend(RabbitTemplate.java:443) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate$1.doInRabbit(RabbitTemplate.java:227) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:408) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.send(RabbitTemplate.java:225) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.convertAndSend(RabbitTemplate.java:242) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
    at org.springframework.amqp.rabbit.core.RabbitTemplate.convertAndSend(RabbitTemplate.java:234) ~[spring-rabbit-1.0.0.RELEASE.jar:na]
...
...
...
Caused by: com.rabbitmq.client.ShutdownSignalException: channel error; reason: {#method<channel.close>(reply-code=404, reply-text=NOT_FOUND - home node 'rabbit@xxx-pf-gs1' of durable queue 'queue' in vhost 'vhost' is down or inaccessible, class-id=50, method-id=10), null, ""}
    at com.rabbitmq.client.impl.ChannelN.asyncShutdown(ChannelN.java:454) ~[amqp-client-2.8.1.jar:na]
    at com.rabbitmq.client.impl.ChannelN.processAsync(ChannelN.java:294) ~[amqp-client-2.8.1.jar:na]
    at com.rabbitmq.client.impl.AMQChannel.handleCompleteInboundCommand(AMQChannel.java:144) ~[amqp-client-2.8.1.jar:na]
    at com.rabbitmq.client.impl.AMQChannel.handleFrame(AMQChannel.java:91) ~[amqp-client-2.8.1.jar:na]
    at com.rabbitmq.client.impl.AMQConnection$MainLoop.run(AMQConnection.java:519) ~[amqp-client-2.8.1.jar:na]

1.2 406 PRECONDITION_FAILED

org.springframework.amqp.AmqpIOException: java.io.IOException
        at org.springframework.amqp.rabbit.support.RabbitExceptionTranslator.convertRabbitAccessException(RabbitExceptionTranslator.java:71) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.RabbitAccessor.convertRabbitAccessException(RabbitAccessor.java:113) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:1461) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:1411) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:1387) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitAdmin.initialize(RabbitAdmin.java:500) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitAdmin$11.onCreate(RabbitAdmin.java:419) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.CompositeConnectionListener.onCreate(CompositeConnectionListener.java:33) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createConnection(CachingConnectionFactory.java:571) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.ConnectionFactoryUtils$1.createConnection(ConnectionFactoryUtils.java:90) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.ConnectionFactoryUtils.doGetTransactionalResourceHolder(ConnectionFactoryUtils.java:140) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.connection.ConnectionFactoryUtils.getTransactionalResourceHolder(ConnectionFactoryUtils.java:76) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.listener.BlockingQueueConsumer.start(BlockingQueueConsumer.java:505) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1382) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_73]
Caused by: java.io.IOException: null
        at com.rabbitmq.client.impl.AMQChannel.wrap(AMQChannel.java:105) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.wrap(AMQChannel.java:101) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.exnWrappingRpc(AMQChannel.java:123) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.ChannelN.queueDeclare(ChannelN.java:925) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.ChannelN.queueDeclare(ChannelN.java:50) ~[amqp-client-3.6.6.jar:3.6.6]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_73]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_73]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_73]
        at java.lang.reflect.Method.invoke(Method.java:497) ~[na:1.8.0_73]
        at org.springframework.amqp.rabbit.connection.CachingConnectionFactory$CachedChannelInvocationHandler.invoke(CachingConnectionFactory.java:955) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at com.sun.proxy.$Proxy45.queueDeclare(Unknown Source) ~[na:na]
        at org.springframework.amqp.rabbit.core.RabbitAdmin.declareQueues(RabbitAdmin.java:575) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitAdmin.access$200(RabbitAdmin.java:66) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitAdmin$12.doInRabbit(RabbitAdmin.java:504) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:1455) ~[spring-rabbit-1.7.1.RELEASE.jar:na]
        ... 12 common frames omitted
Caused by: com.rabbitmq.client.ShutdownSignalException: channel error; protocol method: #method<channel.close>(reply-code=406, reply-text=PRECONDITION_FAILED - inequivalent arg 'x-dead-letter-exchange' for queue 'queue' in vhost '/': received the value 'exchange' of type 'longstr' but current is none, class-id=50, method-id=10)
        at com.rabbitmq.utility.ValueOrException.getValue(ValueOrException.java:66) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.utility.BlockingValueOrException.uninterruptibleGetValue(BlockingValueOrException.java:32) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel$BlockingRpcContinuation.getReply(AMQChannel.java:366) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.privateRpc(AMQChannel.java:229) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.exnWrappingRpc(AMQChannel.java:117) ~[amqp-client-3.6.6.jar:3.6.6]
        ... 24 common frames omitted
Caused by: com.rabbitmq.client.ShutdownSignalException: channel error; protocol method: #method<channel.close>(reply-code=406, reply-text=PRECONDITION_FAILED - inequivalent arg 'x-dead-letter-exchange' for queue 'biliNotifyQueue' in vhost '/': received the value 'sdkNotifyExchange' of type 'longstr' but current is none, class-id=50, method-id=10)
        at com.rabbitmq.client.impl.ChannelN.asyncShutdown(ChannelN.java:483) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.ChannelN.processAsync(ChannelN.java:318) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.handleCompleteInboundCommand(AMQChannel.java:143) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQChannel.handleFrame(AMQChannel.java:90) ~[amqp-client-3.6.6.jar:3.6.6]
        at com.rabbitmq.client.impl.AMQConnection$MainLoop.run(AMQConnection.java:571) ~[amqp-client-3.6.6.jar:3.6.6]
        ... 1 common frames omitted

2.触发场景与解决方案

2.1 404 NOT_FOUND

2.1.1 场景

集群环境下(节点A、B、C)，节点A宕机(QueueA所属节点)，客户端在剩余节点上声明持久化队列QueueA（非镜像队列），服务端响应404 NOT_FOUND。
注意：消费者客户端收到404报错后，默认自动关闭，需编写监听器重连或重启应用恢复。

2.1.2 解决方案

恢复节点A，消费者客户端所在应用重启。
声明QueueA为高可用镜像队列。

2.2 406 PRECONDITION_FAILED

2.2.1 场景

当客户端声明持久化队列QueueB参数与服务端保存参数不一致时，服务端响应406 PRECONDITION_FAILED。声明操作失败。

2.2.2 解决方案

根据实际需求，调整服务端或客户端QueueB参数，使双方匹配。
(rabbitmqctl -p vhostname list_queues name arguments 命令可检查服务器中队列参数)

3.参考资料

《RabbitMQ实战高效部署分布式消息队列》- Alvaro Videla,Jason J.W. Williams 著
RabbitMQ Broker Configuration
High Available(Mirrored) Queue
Clustering Guide
Amqp 0.9.1 Complete Reference Guide