mirror of
https://github.com/dotnet/MQTTnet.git
synced 2026-10-15 00:15:15 +08:00
Christian
e43e7f8e36
* Make use of language features line AsyncDispose, init etc. * Fix build errors * Improve Unit Tests * Improve Unit Tests * Improve Unit Tests * Update samples * Remove obsolete benchmarks * Code cleanup * Improve unit tests * Improve Unit Tests * Fix Unit Tests * Improve Unit Tests * Improve Unit Tests * Restore unit tests * Exclude load tests from unit tests
267 lines
11 KiB
C#
267 lines
11 KiB
C#
// Licensed to the .NET Foundation under one or more agreements.
|
|
// The .NET Foundation licenses this file to you under the MIT license.
|
|
// See the LICENSE file in the project root for more information.
|
|
|
|
namespace MQTTnet.Server.Internal;
|
|
|
|
/*
|
|
* The MqttSubscription object stores subscription parameters and calculates
|
|
* topic hashes.
|
|
*
|
|
* Use of Topic Hashes to improve message processing performance
|
|
* =============================================================
|
|
*
|
|
* Motivation
|
|
* -----------
|
|
* In a typical use case for MQTT there may be many publishers (sensors or
|
|
* other devices in the field) and few subscribers (monitoring all or many topics).
|
|
* Each publisher may have one or more topic(s) to publish and therefore both, the
|
|
* number of publishers and the number of topics may be large.
|
|
*
|
|
* Maintaining subscribers in a separate container
|
|
* -----------------------------------------------
|
|
* Instead of placing all sessions into a single _sessions container, subscribers
|
|
* are added into another _subscriberSessions container (if a client is a
|
|
* subscriber and a publisher then the client is present in both containers). The
|
|
* cost is some additional bookkeeping work upon subscription where each client
|
|
* session needs to maintain a list of subscribed topics.
|
|
*
|
|
* When the client subscribes to the first topic, then the session manager adds
|
|
* the client to the _subscriberSessions container, and when the client
|
|
* unsubscribes from the last topic then the session manager removes the client
|
|
* from the container. Now, when an application message arrives, only the list of
|
|
* subscribers need processing instead of looping through potentially thousands of
|
|
* publishers.
|
|
*
|
|
* Improving subscriber topic lookup
|
|
* ---------------------------------
|
|
* For each subscriber, it needs to be determined whether an application message
|
|
* matches any topic the subscriber has subscribed to. There may only be few
|
|
* subscribers but there may be many subscribed topics, including wildcard topics.
|
|
*
|
|
* The implemented approach uses a topic hash and a hash mask calculated on the
|
|
* subscribed topic and the published topic (the application message topic) to
|
|
* find candidates for a match, with the existing match logic evaluating a reduced
|
|
* number of candidates.
|
|
*
|
|
* For each subscription, the topic hash and a hash mask is stored with the
|
|
* subscription, and for each application message received, the hash is calculated
|
|
* for the published topic before attempting to find matching subscriptions. The
|
|
* hash calculation itself is simple and does not have a large performance impact.
|
|
*
|
|
* We'll first explain how topic hashes and hash masks are constructed and then how
|
|
* they are used.
|
|
*
|
|
* Topic hash
|
|
* ----------
|
|
* Topic hashes are stored as 64-bit numbers. Each byte within the 64-bit number
|
|
* relates to one MQTT topic level. A checksum is calculated for each topic level
|
|
* by iterating over the characters within the topic level (cast to byte) and the
|
|
* result is stored into the corresponding byte of the 64-bit number. If a topic
|
|
* level contains a wildcard character, then 0x00 is stored instead of the
|
|
* checksum.
|
|
*
|
|
* If there are less than 8 levels then the rest of the 64-bit number is filled
|
|
* with 0xff. If there are more than 8 levels then topics where the first 8 MQTT
|
|
* topic levels are identical will have the same hash value.
|
|
*
|
|
* This is the topic hash for the MQTT topic below: 0x655D4AF1FFFFFF
|
|
*
|
|
* client1/building1/level1/sensor1 (empty) (empty) (empty) (empty)
|
|
* \_____/ \_______/ \____/ \_____/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0x65 0x5D 0x4A 0xF1 0xFF 0xFF 0xFF 0xFF
|
|
*
|
|
* This is the topic hash for an MQTT topic containing a wildcard: 0x655D00F1FFFFFF
|
|
*
|
|
* client1/building1/ + /sensor1 (empty) (empty) (empty) (empty)
|
|
* \_____/ \_______/ \_/ \_____/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0x65 0x5D 0 0xF1 0xFF 0xFF 0xFF 0xFF
|
|
*
|
|
* For topics that contain the multi level wildcard # at the end, the topic hash
|
|
* is filled with 0x00: 0x65004A00000000
|
|
*
|
|
* client1/ + /level1/ # (empty) (empty) (empty) (empty)
|
|
* \_____/ \_/ \____/ \_/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0x65 0 0x4A 0 0 0 0 0
|
|
*
|
|
*
|
|
* Topic hash mask
|
|
* ---------------
|
|
* The hash mask simply contains 0xFF for non-wildcard topic levels and 0x00 for
|
|
* wildcard topic levels. Here are the topic hash masks for the examples above.
|
|
*
|
|
* client1/building1/level1/sensor1 (empty) (empty) (empty) (empty)
|
|
* \_____/ \_______/ \____/ \_____/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF
|
|
*
|
|
* client1/building1/ + /sensor1 (empty) (empty) (empty) (empty)
|
|
* \_____/ \_______/ \_/ \_____/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0xFF 0xFF 0 0xFF 0xFF 0xFF 0xFF 0xFF
|
|
*
|
|
* client1/ + /level1/ # (empty) (empty) (empty) (empty)
|
|
* \_____/ \_/ \____/ \_/ \_____/ \_____/ \_____/ \_____/
|
|
* | | | | | | | |
|
|
* 0xFF 0 0xFF 0 0 0 0 0
|
|
*
|
|
*
|
|
* Topic hash and hash mask properties
|
|
* -----------------------------------
|
|
* The following properties of topic hashes and hash masks can be exploited to
|
|
* find potentially matching subscribed topics for a given published topic.
|
|
*
|
|
* (1) If a subscribed topic does not contain wildcards then the topic hash of the
|
|
* subscribed topic must be equal to the topic hash of the published topic,
|
|
* otherwise the subscribed topic cannot be a candidate for a match.
|
|
*
|
|
* (2) If a subscribed topic contains wildcards then the hash of the published
|
|
* topic masked with the subscribed topic's hash mask must be equal to the hash of
|
|
* the subscribed topic. I.e. a subscribed topic is a candidate for a match if:
|
|
* (publishedTopicHash & subscribedTopicHashMask) == subscribedTopicHash
|
|
*
|
|
* (3) If a subscribed topic contains wildcards then any potentially matching
|
|
* published topic must have a hash value that is greater than or equal to the
|
|
* hash value of the subscribed topic (because the subscribed topic contains
|
|
* zeroes in wildcard positions).
|
|
*
|
|
* Match finding
|
|
* -------------
|
|
* The subscription manager maintains two separate dictionaries to assist finding
|
|
* matches using topic hashes: a _noWildcardSubscriptionsByTopicHash dictionary
|
|
* containing all subscriptions that do not have wildcards, and a
|
|
* _wildcardSubscriptionsByTopicHash dictionary containing subscriptions with
|
|
* wildcards.
|
|
*
|
|
* For subscriptions without wildcards, all potential candidates for a match are
|
|
* obtained by a single look-up (exploiting point 1 above).
|
|
*
|
|
* For subscriptions with wildcards, the subscription manager loops through the
|
|
* wildcard subscriptions and selects candidates that satisfy condition
|
|
* (publishedTopicHash & subscribedTopicMask) == subscribedTopicHash (point 2).
|
|
* The loop could exit early if wildcard subscriptions were stored into a sorted
|
|
* dictionary (utilizing point 3), but, after testing, there does not seem to be
|
|
* any real benefit doing so.
|
|
*
|
|
* Other considerations
|
|
* --------------------
|
|
* Characters in the topic string are cast to byte and any additional bytes in a
|
|
* multi-byte character are disregarded. Best guess is that this does not impact
|
|
* performance in practice.
|
|
*
|
|
* Instead of one-byte checksums per topic level, one-word checksums per topic
|
|
* level could be used. If most topics contained four levels or less then hash
|
|
* buckets would be shallower.
|
|
*
|
|
* For very large numbers of topics, performing a parallel search may help further.
|
|
*
|
|
* To also handle a larger number of subscribers, it may be beneficial to maintain
|
|
* a subscribers-by-subscription-topic dictionary.
|
|
*/
|
|
public static class MqttTopicHash
|
|
{
|
|
public static void Calculate(string topic, out ulong resultHash, out ulong resultHashMask, out bool resultHasWildcard)
|
|
{
|
|
// calculate topic hash
|
|
ulong hash = 0;
|
|
ulong hashMaskInverted = 0;
|
|
ulong levelBitMask = 0;
|
|
ulong fillLevelBitMask = 0;
|
|
var hasWildcard = false;
|
|
byte checkSum = 0;
|
|
var level = 0;
|
|
|
|
var i = 0;
|
|
while (i < topic.Length)
|
|
{
|
|
var c = topic[i];
|
|
if (c == MqttTopicFilterComparer.LevelSeparator)
|
|
{
|
|
// done with this level
|
|
hash <<= 8;
|
|
hash |= checkSum;
|
|
hashMaskInverted <<= 8;
|
|
hashMaskInverted |= levelBitMask;
|
|
checkSum = 0;
|
|
levelBitMask = 0;
|
|
++level;
|
|
if (level >= 8)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
else if (c == MqttTopicFilterComparer.SingleLevelWildcard)
|
|
{
|
|
levelBitMask = 0xff;
|
|
hasWildcard = true;
|
|
}
|
|
else if (c == MqttTopicFilterComparer.MultiLevelWildcard)
|
|
{
|
|
// checksum is zero for a valid topic
|
|
levelBitMask = 0xff;
|
|
// fill rest with this fillLevelBitMask
|
|
fillLevelBitMask = 0xff;
|
|
hasWildcard = true;
|
|
break;
|
|
}
|
|
else
|
|
{
|
|
// The checksum should be designed to reduce the hash bucket depth for the expected
|
|
// fairly regularly named MQTT topics that don't differ much,
|
|
// i.e. "room1/sensor1"
|
|
// "room1/sensor2"
|
|
// "room1/sensor3"
|
|
// etc.
|
|
if ((c & 1) == 0)
|
|
{
|
|
checkSum += (byte)c;
|
|
}
|
|
else
|
|
{
|
|
checkSum ^= (byte)(c >> 1);
|
|
}
|
|
}
|
|
|
|
++i;
|
|
}
|
|
|
|
// Shift hash left and leave zeroes to fill ulong
|
|
if (level < 8)
|
|
{
|
|
hash <<= 8;
|
|
hash |= checkSum;
|
|
hashMaskInverted <<= 8;
|
|
hashMaskInverted |= levelBitMask;
|
|
++level;
|
|
while (level < 8)
|
|
{
|
|
hash <<= 8;
|
|
hashMaskInverted <<= 8;
|
|
hashMaskInverted |= fillLevelBitMask;
|
|
++level;
|
|
}
|
|
}
|
|
|
|
if (!hasWildcard)
|
|
{
|
|
while (i < topic.Length)
|
|
{
|
|
var c = topic[i];
|
|
if (c == MqttTopicFilterComparer.SingleLevelWildcard || c == MqttTopicFilterComparer.MultiLevelWildcard)
|
|
{
|
|
hasWildcard = true;
|
|
break;
|
|
}
|
|
|
|
++i;
|
|
}
|
|
}
|
|
|
|
resultHash = hash;
|
|
resultHashMask = ~hashMaskInverted;
|
|
resultHasWildcard = hasWildcard;
|
|
}
|
|
} |