i see so many devices that can easily connect to Azure IoT hub via MQTT. But it is NOT as easy to connect those same devices to Azure IoT Central. Is there a way to send those data from Azure IoT Hub to Azure IoT Central?
In the case of sending only a telemetry data to the Azure IoT Central App, you can use the Azure Event Grid integrator, where the device telemetry message is published via the Azure IoT Hub routing feature:
The following code snippet is an example of the webhook subscriber implementation (HttpTrigger Function) for handling all needs such as DPS, etc.
function.json file:
{
"bindings": [
{
"name": "eventGridEvent",
"authLevel": "function",
"methods": [
"post",
"options"
],
"direction": "in",
"type": "httpTrigger"
},
{
"name": "$return",
"type": "http",
"direction": "out"
}
]
}
run.csx file:
#r "Newtonsoft.Json"
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Primitives;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Net;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
public static async Task<ActionResult> Run(JObject eventGridEvent, HttpRequest req, ILogger log)
{
if (req.Method == HttpMethod.Options.ToString())
{
log.LogInformation("CloudEventSchema validation");
req.HttpContext.Response.Headers.Add("Webhook-Allowed-Origin", req.Headers["WebHook-Request-Origin"].FirstOrDefault()?.Trim());
return (ActionResult)new OkResult();
}
// consumer of telemetry (iot central)
uint sasTokenTTLInHrs = 1;
string iotcScopeId = req.Headers["iotc-scopeId"].FirstOrDefault() ?? Environment.GetEnvironmentVariable("AzureIoTC_scopeId");
string iotcSasToken = req.Headers["iotc-sasToken"].FirstOrDefault() ?? Environment.GetEnvironmentVariable("AzureIoTC_sasToken");
log.LogInformation($"CloudEvent_Id = {eventGridEvent["id"]}");
log.LogInformation($"AzureIoT_scopeId = {iotcScopeId}");
// mandatory properties
string source = eventGridEvent["data"]?["systemProperties"]?["iothub-message-source"]?.Value<string>();
string deviceId = eventGridEvent["data"]?["systemProperties"]?["iothub-connection-device-id"]?.Value<string>();
if (source == "Telemetry" && !string.IsNullOrEmpty(deviceId) && Regex.IsMatch(deviceId, #"^[a-z0-9\-]+$"))
{
var sysProp = eventGridEvent["data"]["systemProperties"];
var appProp = eventGridEvent["data"]["properties"];
// device model
var component = appProp?["iothub-app-component-name"]?.Value<string>() ?? sysProp["dt-subject"]?.Value<string>() ?? "";
var modelId = appProp?["iothub-app-model-id"]?.Value<string>() ?? sysProp["dt-dataschema"]?.Value<string>();
// creation time
var enqueuedtime = sysProp["iothub-enqueuedtime"]?.Value<DateTime>().ToString("o");
var ctime = appProp?["iothub-creation-time-utc"]?.Value<DateTime>().ToString("o");
// device group (device prefix)
var deviceGroup = appProp?["iothub-app-device-group"]?.Value<string>();
deviceId = $"{(deviceGroup == null ? "" : deviceGroup + "-")}{deviceId}";
// remove sysprop
((JObject)eventGridEvent["data"]).Remove("systemProperties");
try
{
var info = await Connectivity.GetConnectionInfo(deviceId, modelId, iotcScopeId, iotcSasToken, log, sasTokenTTLInHrs);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Authorization", info.SasToken);
client.DefaultRequestHeaders.Add("dt-subject", component);
client.DefaultRequestHeaders.Add("iothub-app-iothub-creation-time-utc", ctime ?? enqueuedtime);
var response = await client.PostAsJsonAsync(info.RequestUri, eventGridEvent["data"]);
response.EnsureSuccessStatusCode();
}
log.LogInformation($"POST: {info.RequestUri}\r\n{eventGridEvent["data"]}");
}
catch(Exception ex)
{
log.LogError(ex.InnerException == null ? ex.Message : ex.InnerException.Message);
Connectivity.RemoveDevice(deviceId);
throw ex; // for retrying and deadlettering undeliverable message
}
}
else
{
log.LogWarning($"Wrong event message:\r\n{eventGridEvent}");
}
return (ActionResult)new OkResult();
}
class ConnectivityInfo
{
public string IoTHubName { get; set; }
public string RequestUri { get; set; }
public string SasToken { get; set; }
public ulong SaSExpiry { get; set; }
public string ModelId { get; set; }
public string DeviceConnectionString { get; set; }
}
static class Connectivity
{
static Dictionary<string, ConnectivityInfo> devices = new Dictionary<string, ConnectivityInfo>();
public static async Task<ConnectivityInfo> GetConnectionInfo(string deviceId, string modelId, string iotcScopeId, string iotcSasToken, ILogger log, uint sasTokenTTLInHrs = 24, int retryCounter = 10, int pollingTimeInSeconds = 3)
{
if (devices.ContainsKey(deviceId))
{
if (!string.IsNullOrEmpty(modelId) && devices[deviceId].ModelId != modelId)
{
log.LogWarning($"Reprovissiong device with new model");
devices.Remove(deviceId);
}
else
{
if (!SharedAccessSignatureBuilder.IsValidExpiry(devices[deviceId].SaSExpiry, 100))
{
log.LogWarning($"Refreshing sasToken");
devices[deviceId].SasToken = SharedAccessSignatureBuilder.GetSASTokenFromConnectionString(devices[deviceId].DeviceConnectionString, sasTokenTTLInHrs);
devices[deviceId].SaSExpiry = ulong.Parse(SharedAccessSignatureBuilder.GetExpiry(sasTokenTTLInHrs));
}
return devices[deviceId];
}
}
string deviceKey = SharedAccessSignatureBuilder.ComputeSignature(iotcSasToken, deviceId);
string address = $"https://global.azure-devices-provisioning.net/{iotcScopeId}/registrations/{deviceId}/register?api-version=2021-06-01";
string sas = SharedAccessSignatureBuilder.GetSASToken($"{iotcScopeId}/registrations/{deviceId}", deviceKey, "registration", 1);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Authorization", sas);
client.DefaultRequestHeaders.Add("accept", "application/json");
string jsontext = string.IsNullOrEmpty(modelId) ? null : $"{{ \"modelId\":\"{modelId}\" }}";
var response = await client.PutAsync(address, new StringContent(JsonConvert.SerializeObject(new { registrationId = deviceId, payload = jsontext }), Encoding.UTF8, "application/json"));
var atype = new { errorCode = "", message = "", operationId = "", status = "", registrationState = new JObject() };
do
{
dynamic operationStatus = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), atype);
if (!string.IsNullOrEmpty(operationStatus.errorCode))
{
throw new Exception($"{operationStatus.errorCode} - {operationStatus.message}");
}
response.EnsureSuccessStatusCode();
if (operationStatus.status == "assigning")
{
Task.Delay(TimeSpan.FromSeconds(pollingTimeInSeconds)).Wait();
address = $"https://global.azure-devices-provisioning.net/{iotcScopeId}/registrations/{deviceId}/operations/{operationStatus.operationId}?api-version=2021-06-01";
response = await client.GetAsync(address);
}
else if (operationStatus.status == "assigned")
{
var cinfo = new ConnectivityInfo();
cinfo.ModelId = modelId;
cinfo.IoTHubName = operationStatus.registrationState.assignedHub;
cinfo.DeviceConnectionString = $"HostName={cinfo.IoTHubName};DeviceId={deviceId};SharedAccessKey={deviceKey}";
cinfo.RequestUri = $"https://{cinfo.IoTHubName}/devices/{deviceId}/messages/events?api-version=2021-04-12";
cinfo.SasToken = SharedAccessSignatureBuilder.GetSASToken($"{cinfo.IoTHubName}/{deviceId}", deviceKey, null, sasTokenTTLInHrs);
cinfo.SaSExpiry = ulong.Parse(SharedAccessSignatureBuilder.GetExpiry(sasTokenTTLInHrs));
devices.Add(deviceId, cinfo);
log.LogInformation($"DeviceConnectionString: {cinfo.DeviceConnectionString}");
return cinfo;
}
else
{
throw new Exception($"{operationStatus.registrationState.status}: {operationStatus.registrationState.errorCode} - {operationStatus.registrationState.errorMessage}");
}
} while (--retryCounter > 0);
throw new Exception("Registration device status retry timeout exprired, try again.");
}
}
public static void RemoveDevice(string deviceId)
{
if (devices.ContainsKey(deviceId))
devices.Remove(deviceId);
}
}
public sealed class SharedAccessSignatureBuilder
{
public static string GetHostNameNamespaceFromConnectionString(string connectionString)
{
return GetPartsFromConnectionString(connectionString)["HostName"].Split('.').FirstOrDefault();
}
public static string GetSASTokenFromConnectionString(string connectionString, uint hours = 24)
{
var parts = GetPartsFromConnectionString(connectionString);
if (parts.ContainsKey("HostName") && parts.ContainsKey("SharedAccessKey"))
return GetSASToken(parts["HostName"], parts["SharedAccessKey"], parts.Keys.Contains("SharedAccessKeyName") ? parts["SharedAccessKeyName"] : null, hours);
else
return string.Empty;
}
public static string GetSASToken(string resourceUri, string key, string keyName = null, uint hours = 24)
{
try
{
var expiry = GetExpiry(hours);
string stringToSign = System.Web.HttpUtility.UrlEncode(resourceUri) + "\n" + expiry;
var signature = SharedAccessSignatureBuilder.ComputeSignature(key, stringToSign);
var sasToken = keyName == null ?
String.Format(CultureInfo.InvariantCulture, "SharedAccessSignature sr={0}&sig={1}&se={2}", System.Web.HttpUtility.UrlEncode(resourceUri), System.Web.HttpUtility.UrlEncode(signature), expiry) :
String.Format(CultureInfo.InvariantCulture, "SharedAccessSignature sr={0}&sig={1}&se={2}&skn={3}", System.Web.HttpUtility.UrlEncode(resourceUri), System.Web.HttpUtility.UrlEncode(signature), expiry, keyName);
return sasToken;
}
catch
{
return string.Empty;
}
}
#region Helpers
public static string ComputeSignature(string key, string stringToSign)
{
using (HMACSHA256 hmac = new HMACSHA256(Convert.FromBase64String(key)))
{
return Convert.ToBase64String(hmac.ComputeHash(Encoding.UTF8.GetBytes(stringToSign)));
}
}
public static Dictionary<string, string> GetPartsFromConnectionString(string connectionString)
{
return connectionString.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Split(new[] { '=' }, 2)).ToDictionary(x => x[0].Trim(), x => x[1].Trim(), StringComparer.OrdinalIgnoreCase);
}
// default expiring = 24 hours
public static string GetExpiry(uint hours = 24)
{
TimeSpan sinceEpoch = DateTime.UtcNow - new DateTime(1970, 1, 1);
return Convert.ToString((ulong)sinceEpoch.TotalSeconds + 3600 * hours);
}
public static DateTime GetDateTimeUtcFromExpiry(ulong expiry)
{
return (new DateTime(1970, 1, 1)).AddSeconds(expiry);
}
public static bool IsValidExpiry(ulong expiry, ulong toleranceInSeconds = 0)
{
return GetDateTimeUtcFromExpiry(expiry) - TimeSpan.FromSeconds(toleranceInSeconds) > DateTime.UtcNow;
}
#endregion
}
The following screen snippet shows part of the subscription for passing requested headers for webhook subscriber:
Note, that the mapping feature can be used at the Azure IoT Central App on the input side, based on the device model.
As the above first picture shows, this solution is based on using the Azure Event Grid feature, where the Azure IoT Hub represents a publisher of the device telemetry data and the Azure IoT Central app is its consumer.
The logical connectivity between the Azure IoT Hub and Azure IoT Central is done via the AEG Subscription with a webhook destination handler such as the HttpTrigger Function (see the above implementation).
Note, that this subscription is configured for delivering an event message (device telemetry data) in the CloudEventSchema.
Device that provisions itself in IoTHub via DPS will work with IoT Central with no change other than ID Scope sent by device during provisioning which identifies DPS service instance. One ID Scope will point it specific IoT Hub configured in DPS enrollment group, while other will point it to an internal IoT Hub in IoT Central application (IoT Central spins additional internal IoT Hubs as needed for auto scaling, which is why it has its own internal DPS).
Use of DPS allows provisioning of the device to specific IoTHub at the first call and subsequently the change can be triggered explicitly for reprovisioning to different IoTHub or IoT Central, which can be used to move device if needed. This functionality allows scenarios where you can force a device to connect to an IoT Hub or IoT Central by implementing ID Scope change direct method and triggering reprovisioning. Use of DPS is highly recommended as it simplifies provisioning and provides this flexibility.
Reprovisioning should be part of retry logic on the device in case if it fails to connect to IoTHub for certain amount of time in addition to on-demand change described above.
What makes you think "But it is NOT as easy to connect those same devices to Azure IoT Central"?
Any device connecting to IoTHub can also connect to IoTCentral, you just need to provision the device using DPS, it will get the IoTHub hostname and everything else will work the same way.
I have a project with ASP.NET Core integrated with Redis Sentinel.
Caching works very well with Sentinel but it doesn't work while getting all keys with GetServer(), and it wants me to give it a parameter ServiceName, I don't know how to find it??
There is a Master and 4 Slaves
--> appsettings.json
RedisConfiguration:ConnectionString --- > "127.0.0.1:6379,127.0.0.1:6380,127.0.0.1:6381,127.0.0.1:6382,127.0.0.1:6383"
--> RedisCacheManager.cs
Constructor :
public RedisCacheManager()
{
_connectionString = configuration.GetSection("RedisConfiguration:ConnectionString")?.Value;
var connectionStrings = _connectionString.Split(",");
_configurationOptions = new ConfigurationOptions()
{
EndPoints = { aa.ToString() },
AbortOnConnectFail = false,
AsyncTimeout = 10000,
ConnectTimeout = 10000,
KeepAlive = 180,
//ServiceName = ServiceName
};
foreach (var item in connectionStrings)
{
_configurationOptions.EndPoints.Add(item);
}
_client = ConnectionMultiplexer.Connect(_configurationOptions);
}
--> RemoveByPattern method in RedisCacheManager.cs
public void RemoveByPattern()
{
//THIS IS WHERE I USE SENTINEL CONNECT AND CONFIG OPTIONS....
//var aa = ConnectionMultiplexer.SentinelConnect(_configurationOptions);
//THIS IS ALSO WHERE I NEED SERVICE NAME... (FOUND THIS WHILE RESEARCHING)
//var settings = ConfigurationOptions.Parse("localhost,serviceName=mymaster");
// HERE I HAVE TO USE ONLY ONE ENDPOINT....
var server = ConnectionMultiplexer.Connect(settings).GetServer(_client.GetEndPoints().First());
var keys = server.Keys();
var values = keys.Where(x => x.ToString().Contains(pattern)).Select(c => (string)c);
List<string> listKeys = new();
listKeys.AddRange(values);
foreach (var key in listKeys)
{
await _client.GetDatabase().KeyDeleteAsync(key);
}
}
I'm using Apache Ignite on Azure Kubernetes as a distributed cache. Also, I have a web API on Azure based on .NET6.
I connect to Ignite with IgniteClient class. I made it a singleton but the connection closes in 5 seconds after starting.
I've tried
ReconnectDisabled = false
AND
SocketTimeout = TimeSpan.FromMilliseconds(System.Threading.Timeout.Infinite)
but both of them didn't work. How can I keep alive the connection all the time as a singleton?
Here is my configuration code of the Ignite;
public CacheManager()
{
ConnectIgnite();
}
public void ConnectIgnite()
{
_ignite = Ignition.StartClient(GetIgniteConfiguration());
}
public IgniteClientConfiguration GetIgniteConfiguration()
{
var appSettingsJson = AppSettingsJson.GetAppSettings();
var igniteEndpoints = appSettingsJson["AppSettings:IgniteEndpoint"];
var igniteUser = appSettingsJson["AppSettings:IgniteUser"];
var ignitePassword = appSettingsJson["AppSettings:IgnitePassword"];
var nodeList = igniteEndpoints.Split(",");
var config = new IgniteClientConfiguration
{
Endpoints = nodeList,
UserName = igniteUser,
Password = ignitePassword,
EnablePartitionAwareness = true,
SocketTimeout = TimeSpan.FromMilliseconds(System.Threading.Timeout.Infinite)
};
return config;
}
We have implemented Azure key vault in the .NET core application. Everything is working fine until we disabled the secret from the list - After my application tries to fetch the list again it started giving me the exception
Unhandled exception. Microsoft.Azure.KeyVault.Models.KeyVaultErrorException: Operation get is not allowed on a disabled secret.
at Microsoft.Azure.KeyVault.KeyVaultClient.GetSecretWithHttpMessagesAsync(String vaultBaseUrl, String secretName, String secretVersion, Dictionary`2 customHeaders, CancellationToken cancellationToken)
at Microsoft.Azure.KeyVault.KeyVaultClientExtensions.GetSecretAsync(IKeyVaultClient operations, String secretIdentifier, CancellationToken cancellationToken)
at Microsoft.Extensions.Configuration.AzureKeyVault.AzureKeyVaultConfigurationProvider.LoadAsync()
at Microsoft.Extensions.Configuration.AzureKeyVault.AzureKeyVaultConfigurationProvider.Load()
at Microsoft.Extensions.Configuration.ConfigurationRoot..ctor(IList`1 providers)
at Microsoft.Extensions.Configuration.ConfigurationBuilder.Build()
at Microsoft.Extensions.Hosting.HostBuilder.BuildAppConfiguration()
at Microsoft.Extensions.Hosting.HostBuilder.Build()
at Vodafone.LandingPage.Program.Main(String[] args) in D:\a\1\s\src\LandingPage\Program.cs:line 30
Code I use to connect with Key Vault in program.cs file.
if (ctx.HostingEnvironment.IsProduction())
{
var builtConfig = builder.Build();
var keyVaultEndpoint = $"https://{builtConfig["AppSettings:KeyVaultName"]}.vault.azure.net/";
var azureServiceTokenProvider = new AzureServiceTokenProvider();
var keyVaultClient = new KeyVaultClient(
new KeyVaultClient.AuthenticationCallback(
azureServiceTokenProvider.KeyVaultTokenCallback));
builder.AddAzureKeyVault(keyVaultEndpoint, keyVaultClient, new DefaultKeyVaultSecretManager());
}
How we can restrict the list so that it will not take the disabled secrets together.
I am using "Get" and "List" permission.
After a research I found below solution.
You can use it like this
Problem : Code which read all secret
builder.AddAzureKeyVault(keyVaultEndpoint, keyVaultClient, new DefaultKeyVaultSecretManager());
Solution : Code Which read only enabled secrets
builder.AddAzureKeyVault(keyVaultEndpoint,keyVaultClient,new PrefixKeyVaultSecretManager(keyVaultEndpoint));
Implementation of IKeyVaultSecretManager
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.Azure.KeyVault;
using Microsoft.Azure.KeyVault.Models;
using Microsoft.Azure.Services.AppAuthentication;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.AzureKeyVault;
namespace KeyVaultPOC
{
public class PrefixKeyVaultSecretManager : IKeyVaultSecretManager
{
private readonly IList<string> _overrides = new List<string>();
public PrefixKeyVaultSecretManager(string vaultUrl)
{
Task.Run(() => LoadListOfOverrides(vaultUrl)).Wait();
}
private async Task LoadListOfOverrides(string vaultUrl)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
var keyVaultClient = new KeyVaultClient(
new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback)
);
var secrets = await keyVaultClient.GetSecretsAsync(vaultUrl);
bool moreSecrets;
do
{
foreach (var secret in secrets)
{
if ((bool)secret.Attributes.Enabled)
{
_overrides.Add(secret.Identifier.Name);
}
}
moreSecrets = !string.IsNullOrEmpty(secrets.NextPageLink);
if (moreSecrets)
{
secrets = await keyVaultClient.GetSecretsNextAsync(secrets.NextPageLink);
}
} while (moreSecrets);
}
public bool Load(SecretItem secret)
{
return true;
}
public string GetKey(SecretBundle secret)
{
var key = secret.SecretIdentifier.Name;
return key;
}
}
}
Ref : https://gist.github.com/davidxcheng/0576659d2c876d299619d979767dcdd6
yml string concatenate does not work with .NET applications.I have tried by removing '$' sign, but it is still not working(Java application uses $ sign - Working fine with Java apps). It is working fine for a single value, but not with concatenation.
yml-01
cicd:
dbname: 172.10.10.110
port: 5432
yml-02
datasource:
url: jdbc:postgresql://${cicd:dbname}:${cicd:port}/sample-db
A solution for placeholder resolution in .NET Configuration (similar to that provided by spring) is available in Steeltoe.Common. We haven't added WebHostBuilder or IConfigurationBuilder extensions just yet, but if you add a recent reference to Steeltoe.Common from the Steeltoe Dev feed you should be able to do something like this:
public static IWebHostBuilder ResolveConfigurationPlaceholders(this IWebHostBuilder hostBuilder, LoggerFactory loggerFactory = null)
{
return hostBuilder.ConfigureAppConfiguration((builderContext, config) =>
{
config.AddInMemoryCollection(PropertyPlaceholderHelper.GetResolvedConfigurationPlaceholders(config.Build(), loggerFactory?.CreateLogger("Steeltoe.Configuration.PropertyPlaceholderHelper")));
});
}
The code above is used in the Steeltoe fork of eShopOnContainers
You should take a look at YamlDotNet.
Here's an example of how to solve your problem using that lib
using YamlDotNet.RepresentationModel;
using YamlDotNet.Core;
Then in your method
var dbname = "172.10.10.110";
var port = "5432";
string content;
using (var reader = new StreamReader("your yml file"))
{
content = reader.ReadToEnd();
}
var doc = new StringReader(content);
var yaml = new YamlStream();
yaml.Load(doc);
// Add the url where you use string interpolation to replace the values
var ymlFile = (YamlMappingNode)yaml.Documents[0].RootNode;
ymlFile.Children["datasource"] = new YamlMappingNode
{
{ "url", $"jdbc:postgresql://{dbname}:{port}/sample-db" }
};
yaml.Save(File.CreateText("C:\\yourNewFile.yml"), assignAnchors: false);
Here's a link to the NetCore package
I've solved this by writing an extension method to the IConfiguration interface.
public static string ReadFromConfigRepo(this IConfiguration configuration, string key)
{
var pattern = #"\{(.*?)\}";
var query = configuration[key];
if (query.Contains('{'))
{
var matches = Regex.Matches(query, pattern);
string value;
foreach (Match m in matches)
{
value = configuration[m.Value.Substring(1, m.Value.Length - 2)];
query = query.Replace(m.Value, value);
}
}
return query.Trim();
}