Topic template extension maintenance (#2022)

* Topic templates: Ensure parameter values are checked for wildcard

* Topic templates: Complement extension methods

- add template.BuildMessage()
- add subscribeOptionsBuilder.WithTopicTemplate()

* Topic templates: switch some samples to use a topic template

The changes should be self-explanatory and are
intended to provide more examples.

* Topic templates: add README

Samples/Client/Client_Subscribe_Samples.cs

@@ -8,6 +8,7 @@
 // ReSharper disable UnusedMember.Local
 
 using MQTTnet.Client;
+using MQTTnet.Extensions.TopicTemplate;
 using MQTTnet.Packets;
 using MQTTnet.Protocol;
 using MQTTnet.Samples.Helpers;
@@ -16,6 +17,8 @@ namespace MQTTnet.Samples.Client;
 
 public static class Client_Subscribe_Samples
 {
+    static MqttTopicTemplate sampleTemplate = new MqttTopicTemplate("mqttnet/samples/topic/{id}");
+    
     public static async Task Handle_Received_Application_Message()
     {
         /*
@@ -42,11 +45,7 @@ public static class Client_Subscribe_Samples
             await mqttClient.ConnectAsync(mqttClientOptions, CancellationToken.None);
 
             var mqttSubscribeOptions = mqttFactory.CreateSubscribeOptionsBuilder()
-                .WithTopicFilter(
+                .WithTopicTemplate(sampleTemplate.WithParameter("id", "2"))
-                    f =>
-                    {
-                        f.WithTopic("mqttnet/samples/topic/2");
-                    })
                 .Build();
 
             await mqttClient.SubscribeAsync(mqttSubscribeOptions, CancellationToken.None);
@@ -88,11 +87,8 @@ public static class Client_Subscribe_Samples
             await mqttClient.ConnectAsync(mqttClientOptions, CancellationToken.None);
 
             var mqttSubscribeOptions = mqttFactory.CreateSubscribeOptionsBuilder()
-                .WithTopicFilter(
+                .WithTopicTemplate(
-                    f =>
+                    sampleTemplate.WithParameter("id", "1"))
-                    {
-                        f.WithTopic("mqttnet/samples/topic/1");
-                    })
                 .Build();
 
             var response = await mqttClient.SubscribeAsync(mqttSubscribeOptions, CancellationToken.None);
@@ -121,21 +117,12 @@ public static class Client_Subscribe_Samples
             // Create the subscribe options including several topics with different options.
             // It is also possible to all of these topics using a dedicated call of _SubscribeAsync_ per topic.
             var mqttSubscribeOptions = mqttFactory.CreateSubscribeOptionsBuilder()
-                .WithTopicFilter(
+                .WithTopicTemplate(
-                    f =>
+                    sampleTemplate.WithParameter("id", "1"))
-                    {
+                .WithTopicTemplate(
-                        f.WithTopic("mqttnet/samples/topic/1");
+                    sampleTemplate.WithParameter("id", "2"), noLocal: true)
-                    })
+                .WithTopicTemplate(
-                .WithTopicFilter(
+                    sampleTemplate.WithParameter("id", "3"), retainHandling: MqttRetainHandling.SendAtSubscribe)
-                    f =>
-                    {
-                        f.WithTopic("mqttnet/samples/topic/2").WithNoLocal();
-                    })
-                .WithTopicFilter(
-                    f =>
-                    {
-                        f.WithTopic("mqttnet/samples/topic/3").WithRetainHandling(MqttRetainHandling.SendAtSubscribe);
-                    })
                 .Build();
 
             var response = await mqttClient.SubscribeAsync(mqttSubscribeOptions, CancellationToken.None);
@@ -162,11 +149,7 @@ public static class Client_Subscribe_Samples
             await mqttClient.ConnectAsync(mqttClientOptions, CancellationToken.None);
 
             var mqttSubscribeOptions = mqttFactory.CreateSubscribeOptionsBuilder()
-                .WithTopicFilter(
+                .WithTopicTemplate(sampleTemplate.WithParameter("id", "1"))
-                    f =>
-                    {
-                        f.WithTopic("mqttnet/samples/topic/1");
-                    })
                 .Build();
 
             var response = await mqttClient.SubscribeAsync(mqttSubscribeOptions, CancellationToken.None);

Samples/MQTTnet.Samples.csproj

@@ -17,6 +17,7 @@
         <ProjectReference Include="..\Source\MQTTnet.AspnetCore\MQTTnet.AspNetCore.csproj" />
         <ProjectReference Include="..\Source\MQTTnet.Extensions.ManagedClient\MQTTnet.Extensions.ManagedClient.csproj" />
         <ProjectReference Include="..\Source\MQTTnet.Extensions.Rpc\MQTTnet.Extensions.Rpc.csproj" />
+        <ProjectReference Include="..\Source\MQTTnet.Extensions.TopicTemplate\MQTTnet.Extensions.TopicTemplate.csproj" />
         <ProjectReference Include="..\Source\MQTTnet.Extensions.WebSocket4Net\MQTTnet.Extensions.WebSocket4Net.csproj" />
         <ProjectReference Include="..\Source\MQTTnet\MQTTnet.csproj" />
     </ItemGroup>

Source/MQTTnet.Extensions.TopicTemplate/MQTTnet.Extensions.TopicTemplate.csproj

@@ -11,10 +11,10 @@
         <Company>The contributors of MQTTnet</Company>
         <Product>MQTTnet</Product>
         <Description>
-           This is an extension library which provides mqtt topic templating logic to support dispatch,
+           Provides mqtt topic templating logic to support dispatch,
-           routing and similar functionality based on the well known moustache syntax, aka
+           routing and similar functionality based on the well known moustache syntax (AsyncAPI compatible).
-           AsyncAPI dynamic channel address.
         </Description>
+		<PackageReadmeFile>README.md</PackageReadmeFile>
         <Authors>The contributors of MQTTnet</Authors>
         <PackageId>MQTTnet.Extensions.TopicTemplate</PackageId>
         <SignAssembly>false</SignAssembly>
@@ -51,6 +51,10 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <None Include="README.md">
+            <Pack>True</Pack>
+            <PackagePath>\</PackagePath>
+        </None>
         <None Include="..\..\Images\nuget.png">
             <Pack>True</Pack>
             <PackagePath>\</PackagePath>

Source/MQTTnet.Extensions.TopicTemplate/MqttTopicTemplate.cs

@@ -16,8 +16,8 @@ namespace MQTTnet.Extensions.TopicTemplate
     ///     segments in curly braces called parameters. This well-known
     ///     'moustache' syntax also matches AsyncAPI Channel Address Expressions.
     ///     The topic template is designed to support dynamic subscription/publication,
-    ///     message-topic matching and routing. It is intended to be more convenient
+    ///     message-topic matching and routing. It is intended to be more safe and
-    ///     than String.Format() for aforementioned purposes.
+    ///     convenient than String.Format() for aforementioned purposes.
     /// </summary>
     /// <example>
     ///     topic/subtopic/{parameter}/{otherParameter}
@@ -282,12 +282,17 @@ namespace MQTTnet.Extensions.TopicTemplate
         /// <returns>the topic template (without the parameter)</returns>
         public MqttTopicTemplate WithoutParameter(string parameter)
         {
-            return WithParameter(parameter, MqttTopicFilterComparer.SingleLevelWildcard.ToString());
+            if (string.IsNullOrEmpty(parameter) || !_parameterSegments.Contains(parameter))
+            {
+                throw new ArgumentException("topic template parameter must exist.");
+            }
+            
+            return ReplaceInternal(parameter, MqttTopicFilterComparer.SingleLevelWildcard.ToString());
         }
         
         /// <summary>
         ///     Substitute a parameter with a given value, thus removing the parameter. If the parameter is not present,
-        ///     the method trows. The value must not contain slashes.
+        ///     the method trows. The value must not contain slashes or wildcards.
         /// </summary>
         /// <param
         ///     name="parameter">
@@ -304,12 +309,19 @@ namespace MQTTnet.Extensions.TopicTemplate
         /// <returns>the topic template (without the parameter)</returns>
         public MqttTopicTemplate WithParameter(string parameter, string value)
         {
-            if (value == null || string.IsNullOrEmpty(parameter) || !_parameterSegments.Contains(parameter) || value.Contains(MqttTopicFilterComparer.LevelSeparator) ||
+            if (value == null || string.IsNullOrEmpty(parameter) || !_parameterSegments.Contains(parameter) ||
+                value.Contains(MqttTopicFilterComparer.LevelSeparator) ||
+                value.Contains(MqttTopicFilterComparer.SingleLevelWildcard) ||
                 value.Contains(MqttTopicFilterComparer.MultiLevelWildcard))
             {
-                throw new ArgumentException("parameter must exist and value must not contain slashes.");
+                throw new ArgumentException("parameter must exist and value must not contain slashes or wildcard.");
             }
             
+            return ReplaceInternal(parameter, value);
+        }
+        
+        private MqttTopicTemplate ReplaceInternal(string parameter, string value)
+        {
             var moustache = "{" + parameter + "}";
             return new MqttTopicTemplate(Template.Replace(moustache, value));
         }

Source/MQTTnet.Extensions.TopicTemplate/README.md

@@ -0,0 +1,79 @@
+# MQTTNet.Extensions.TopicTemplate
+
+Provides mqtt topic templating logic to support dispatch, routing and
+similar functionality based on the well known moustache syntax, aka
+AsyncAPI dynamic channel address.
+
+Generating and parsing MQTT topics and topic filters is often done in
+an ad-hoc fashion, leading to error-prone code and subtly introduced
+showstoppers like forgetting to strip a slash from a parameter.  To
+remedy, this extension aids you write safe, maintainable logic
+for topic generation, routing and dispatch.
+
+
+## How it works
+
+The package lets you write MQTTNet code like:
+```csharp
+var template = new MqttTopicTemplate("App/v1/{sender}/message");
+
+var filter = new MqttTopicFilterBuilder()
+    .WithTopicTemplate(template) // (1)
+    .WithAtLeastOnceQoS()
+    .WithNoLocal()
+    .Build();
+
+// filter.Topic == "App/v1/+/message"
+
+var myTopic = template
+    .WithParameter("sender", "me, myself & i"); // (2)
+
+var message = new MqttApplicationMessageBuilder()
+    .WithTopicTemplate(  // (3)
+        myTopic)
+    .WithPayload("Hello!")
+    .Build();
+
+// message.Topic == "App/v1/me, myself & i/message"
+
+Assert.IsTrue(message.MatchesTopicTemplate(template)); // (4)
+```
+
+At (1), the parameter "sender" was automatically replaced by a plus sign
+(single level wildcard) because the caller is creating a topic filter.
+
+Using the same template (they are immutable), you can send
+a message. At (2), we replace the parameter with a known value.
+If that value contained a level separator or topic filter only
+characters, we'd get an ArgumentException. If we forgot to replace
+the parameter "sender" (e.g. when placing template not myTopic),
+we would be prevented from sending at (3).
+
+On the receiving side, a message can be matched to a template
+using code like at (4).
+
+These daily tasks become much safer and easier to code than 
+string.Format() and Split() allow for. More elaborate computation is supported
+too. You could extract the sender using the template or route
+messages based on topic content.
+
+## Features
+
+- Safe handling of topic parameters in moustache syntax
+- Simplifies topic generation and parsing logic
+- Match topics and extract the values of their parameters
+- Translate between topic templates to ease dynamic routing
+- Derive a canonical topic filter from a bunch of templates
+- Integrates with MQTTNet message, subscription and filter builders
+- Extension method friendly (sealed class with one public ctor)
+
+## Additional documentation
+
+- Some of the [unit tests](https://github.com/dotnet/MQTTnet/blob/master/Source/MQTTnet.Tests/Extensions/MqttTopicTemplate_Tests.cs) are instructive
+- Some examples feature the extension, e.g. TopicGenerator, Client_Subscribe_Samples
+
+## Feedback
+
+Report any issues with the main project on https://github.com/dotnet/MQTTnet/issues
+
+

Source/MQTTnet.Extensions.TopicTemplate/TopicTemplateExtensions.cs

@@ -4,6 +4,8 @@
 
 using System;
 using System.Linq;
+using MQTTnet.Client;
+using MQTTnet.Packets;
 using MQTTnet.Protocol;
 
 namespace MQTTnet.Extensions.TopicTemplate
@@ -55,6 +57,24 @@ namespace MQTTnet.Extensions.TopicTemplate
             return new MqttTopicFilterBuilder().WithTopicTemplate(topicTemplate, subscribeTreeRoot);
         }
 
+        /// <summary>
+        ///     Create a message builder from this template. The template must not have
+        ///     remaining parameters. 
+        /// </summary>
+        /// <param
+        ///     name="topicTemplate">
+        ///     a parameterless topic template
+        /// </param>
+        /// <returns>a new message builder</returns>
+        /// <exception
+        ///     cref="ArgumentException">
+        ///     if the topic template has parameters
+        /// </exception>
+        public static MqttApplicationMessageBuilder BuildMessage(this MqttTopicTemplate topicTemplate)
+        {
+            return new MqttApplicationMessageBuilder().WithTopicTemplate(topicTemplate);
+        }
+        
         /// <summary>
         ///     Return a message builder to respond to this message. The
         ///     message's response topic and correlation data are included
@@ -116,6 +136,29 @@ namespace MQTTnet.Extensions.TopicTemplate
             return builder.WithTopic(subscribeTreeRoot ? topicTemplate.TopicTreeRootFilter : topicTemplate.TopicFilter);
         }
 
+        /// <summary>
+        /// Set the subscription to the template's topic filter.
+        /// </summary>
+        /// <returns>the builder</returns>
+        public static MqttClientSubscribeOptionsBuilder WithTopicTemplate(
+            this MqttClientSubscribeOptionsBuilder builder,
+            MqttTopicTemplate topicTemplate,
+            MqttQualityOfServiceLevel qualityOfServiceLevel = MqttQualityOfServiceLevel.AtMostOnce,
+            bool noLocal = false,
+            bool retainAsPublished = false,
+            MqttRetainHandling retainHandling = MqttRetainHandling.SendAtSubscribe)
+        {
+            return builder.WithTopicFilter(
+                new MqttTopicFilter
+                {
+                    Topic = topicTemplate.TopicFilter,
+                    QualityOfServiceLevel = qualityOfServiceLevel,
+                    NoLocal = noLocal,
+                    RetainAsPublished = retainAsPublished,
+                    RetainHandling = retainHandling
+                });
+        }
+        
         /// <summary>
         ///     Set the publication topic according to the topic template. The template
         ///     must not have remaining (unset) parameters or contain wildcards.

Source/MQTTnet.TestApp/MQTTnet.TestApp.csproj

@@ -21,6 +21,7 @@
         <ProjectReference Include="..\..\Source\MQTTnet.Extensions.ManagedClient\MQTTnet.Extensions.ManagedClient.csproj" />
         <ProjectReference Include="..\..\Source\MQTTnet.Extensions.WebSocket4Net\MQTTnet.Extensions.WebSocket4Net.csproj" />
         <ProjectReference Include="..\..\Source\MQTTnet\MQTTnet.csproj" />
+        <ProjectReference Include="..\MQTTnet.Extensions.TopicTemplate\MQTTnet.Extensions.TopicTemplate.csproj" />
     </ItemGroup>
 
 </Project>

Source/MQTTnet.TestApp/TopicGenerator.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
+using MQTTnet.Extensions.TopicTemplate;
 
 namespace MQTTnet.TestApp
 {
@@ -17,6 +17,8 @@ namespace MQTTnet.TestApp
             singleWildcardTopicsByPublisher = new Dictionary<string, List<string>>();
             multiWildcardTopicsByPublisher = new Dictionary<string, List<string>>();
 
+            MqttTopicTemplate baseTemplate = new MqttTopicTemplate("{publisher}/{building}/{level}/{sensor}");
+
             // Find some reasonable distribution across three topic levels
             var topicsPerLevel = (int)Math.Pow(numTopicsPerPublisher, (1.0 / 3.0));
 
@@ -40,25 +42,37 @@ namespace MQTTnet.TestApp
                 int publisherTopicCount = 0;
 
                 var publisherName = "pub" + p;
+                var publisherTemplate = baseTemplate
+                    .WithParameter("publisher", publisherName);
                 for (var l1 = 0; l1 < numLevel1Topics; ++l1)
                 {
+                    var l1Template = publisherTemplate
+                        .WithParameter("building", "building" + (l1 + 1));
                     for (var l2 = 0; l2 < numLevel2Topics; ++l2)
                     {
+                        var l2Template = l1Template
+                            .WithParameter("level", "level" + (l2 + 1));
                         for (var l3 = 0; l3 < maxNumLevel3Topics; ++l3)
                         {
                             if (publisherTopicCount >= numTopicsPerPublisher)
                                 break;
 
-                            var topic = string.Format("{0}/building{1}/level{2}/sensor{3}", publisherName, l1 + 1, l2 + 1, l3 + 1);
+                            var topic = l2Template
+                                .WithParameter("sensor", "sensor" + (l3 + 1))
+                                .TopicFilter;
                             AddPublisherTopic(publisherName, topic, topicsByPublisher);
 
                             if (l2 == 0)
                             {
-                                var singleWildcardTopic = string.Format("{0}/building{1}/+/sensor{2}", publisherName, l1 + 1, l3 + 1);
+                                var singleWildcardTopic = l1Template
+                                        .WithParameter("sensor", "sensor" + (l3 + 1))
+                                        .TopicFilter;
                                 AddPublisherTopic(publisherName, singleWildcardTopic, singleWildcardTopicsByPublisher);
                                 if (l1 == 0)
                                 {
-                                    var multiWildcardTopic = string.Format("{0}/+/level{1}/+", publisherName, l3 + 1);
+                                    var multiWildcardTopic = publisherTemplate
+                                        .WithParameter("sensor", "sensor" + (l3 + 1))
+                                        .TopicFilter;
                                     AddPublisherTopic(publisherName, multiWildcardTopic, multiWildcardTopicsByPublisher);
                                 }
                             }

Source/MQTTnet.Tests/Extensions/MqttTopicTemplate_Tests.cs

@@ -45,6 +45,29 @@ namespace MQTTnet.Tests.Extensions
             var template = new MqttTopicTemplate("A/B/{foo}/D");
             template.WithParameter("foo", "a#");
         }
+
+        [TestMethod]
+        [ExpectedException(typeof(ArgumentException))]
+        public void RejectsReservedChars2()
+        {
+            var template = new MqttTopicTemplate("A/B/{foo}/D");
+            template.WithParameter("foo", "a+b");
+        }
+
+        [TestMethod]
+        [ExpectedException(typeof(ArgumentException))]
+        public void RejectsReservedChars3()
+        {
+            var template = new MqttTopicTemplate("A/B/{foo}/D");
+            template.WithParameter("foo", "a/b");
+        }
+        
+        [TestMethod]
+        public void AcceptsEmptyValue()
+        {
+            var template = new MqttTopicTemplate("A/B/{foo}/D");
+            template.WithParameter("foo", "");
+        }
         
         [TestMethod]
         [ExpectedException(typeof(MqttProtocolViolationException))]
@@ -60,13 +83,6 @@ namespace MQTTnet.Tests.Extensions
             var _ = new MqttTopicTemplate(null);
         }
 
-        [TestMethod]
-        [ExpectedException(typeof(ArgumentException))]
-        public void RejectsReservedChars2()
-        {
-            var template = new MqttTopicTemplate("A/B/{foo}/D");
-            template.WithParameter("foo", "e/f");
-        }
         
         [TestMethod]
         public void IgnoresEmptyParameters()
@@ -114,6 +130,50 @@ namespace MQTTnet.Tests.Extensions
             Assert.AreEqual("A/v1/+/F", filter.Topic);
         }
 
+        [TestMethod]
+        public void SubscriptionSupport2()
+        {
+            var template = new MqttTopicTemplate("A/v1/{param}/F");
+            
+            var subscribeOptions = new MqttFactory().CreateSubscribeOptionsBuilder()
+                .WithTopicTemplate(template)
+                .WithSubscriptionIdentifier(5)
+                .Build();
+            Assert.AreEqual("A/v1/+/F", subscribeOptions.TopicFilters[0].Topic);
+        }
+
+        [TestMethod]
+        public void SendAndSubscribeSupport()
+        {
+            var template = new MqttTopicTemplate("App/v1/{sender}/message");
+
+            var filter = new MqttTopicFilterBuilder()
+                .WithTopicTemplate(template)
+                .WithAtLeastOnceQoS()
+                .WithNoLocal()
+                .Build();
+
+            Assert.AreEqual("App/v1/+/message", filter.Topic);
+
+            var myTopic = template.WithParameter("sender", "me, myself & i");
+
+            var message = new MqttApplicationMessageBuilder()
+                .WithTopicTemplate(
+                    myTopic)
+                .WithPayload("Hello!")
+                .Build();
+
+            Assert.IsTrue(message.MatchesTopicTemplate(template));
+        }
+
+        [TestMethod]
+        public void SendAndSubscribeSupport2()
+        {
+            var template = new MqttTopicTemplate("App/v1/{sender}/message");
+            Assert.ThrowsException<ArgumentException>(() => 
+                template.BuildMessage());
+        }
+
         [TestMethod]
         public void CanonicalPrefixFilter()
         {

Source/MQTTnet.Tests/TopicGenerator.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using MQTTnet.Extensions.TopicTemplate;
 
 namespace MQTTnet.Tests
 {
@@ -31,31 +32,43 @@ namespace MQTTnet.Tests
             {
                 maxNumLevel3Topics = 1;
             }
+            
+            MqttTopicTemplate baseTemplate = new MqttTopicTemplate("{publisher}/{building}/{level}/{sensor}");
 
             for (var p = 0; p < numPublishers; ++p)
             {
                 int publisherTopicCount = 0;
                 var publisherName = "pub" + p;
+                var publisherTemplate = baseTemplate
+                    .WithParameter("publisher", publisherName);
+                
                 for (var l1 = 0; l1 < numLevel1Topics; ++l1)
                 {
+                    var l1Template = publisherTemplate.WithParameter("building", "building" + (l1 + 1));
                     for (var l2 = 0; l2 < numLevel2Topics; ++l2)
                     {
+                        var l2Template = l1Template.WithParameter("level", "level" + (l2 + 1));
                         for (var l3 = 0; l3 < maxNumLevel3Topics; ++l3)
                         {
                             if (publisherTopicCount >= numTopicsPerPublisher)
                                 break;
 
-                            var topic = string.Format("{0}/building{1}/level{2}/sensor{3}", publisherName, l1 + 1, l2 + 1, l3 + 1);
+                            var l3Template = l2Template
-                            AddPublisherTopic(publisherName, topic, topicsByPublisher);
+                                .WithParameter("sensor", "sensor" + (l3 + 1));
+                            AddPublisherTopic(publisherName, l3Template.TopicFilter, topicsByPublisher);
 
                             if (l2 == 0)
                             {
-                                var singleWildcardTopic = string.Format("{0}/building{1}/+/sensor{2}", publisherName, l1 + 1, l3 + 1);
+                                var singleWildcardTopic = l1Template
+                                    .WithParameter("sensor", "sensor" + (l3 + 1))
+                                    .TopicFilter;
                                 AddPublisherTopic(publisherName, singleWildcardTopic, singleWildcardTopicsByPublisher);
                             }
                             if ((l1 == 0) && (l3 == 0))
                             {
-                                var multiWildcardTopic = string.Format("{0}/+/level{1}/+", publisherName, l2 + 1);
+                                var multiWildcardTopic = publisherTemplate
+                                    .WithParameter("level", "level" + (l2 + 1))
+                                    .TopicFilter;
                                 AddPublisherTopic(publisherName, multiWildcardTopic, multiWildcardTopicsByPublisher);
                             }