OTEL .NET Runtime Instrumentation

.gitignore

@@ -42,3 +42,5 @@
 
 /packages/*
 /Distribution/Configuration/server-access.json
+
+/grafana-storage/*

Projects/Server/Main.cs

@@ -420,6 +420,8 @@ public static class Core
 
         VerifySerialization();
 
+        Telemetry.Start();
+
         _now = DateTime.UtcNow;
         _firstTick = _tickCount = GetTimestamp();
 

Projects/Server/Mobiles/Mobile.cs

@@ -28,6 +28,7 @@ using Server.Targeting;
 using Server.Text;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Runtime.CompilerServices;
 using CalcMoves = Server.Movement.Movement;
 
@@ -196,6 +197,10 @@ public delegate int AOSStatusHandler(Mobile from, int index);
 /// </summary>
 public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPropertyListEntity, IValueLinkListNode<Mobile>
 {
+    private static readonly Counter<long> mobilesCreatedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_created_count");
+    private static readonly Counter<long> mobilesDeletedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_deleted_count");
+    private static readonly Counter<long> mobilesKilledCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_killed_count");
+
     // Allow four warmode changes in 0.5 seconds, any more will be delay for two seconds
     private const int WarmodeCatchCount = 4;
 
@@ -364,6 +369,11 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
         DefaultMobileInit();
 
         World.AddEntity(this);
+
+        mobilesCreatedCounter.Add(1, new KeyValuePair<string, object?>[]
+        {
+            new("Map", Map?.Name),
+        });
     }
 
     public Mobile(Serial serial)
@@ -2419,6 +2429,14 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
 
         DropHolding();
 
+        if (Alive) {
+            mobilesDeletedCounter.Add(1, new KeyValuePair<string, object?>[]
+            {
+                new("Map", Map?.Name),
+                new("Region", Region?.Name),
+            });
+        }
+
         Region.OnRegionChange(this, m_Region, null);
 
         m_Region = null;
@@ -4801,6 +4819,13 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
             }
         }
 
+        mobilesKilledCounter.Add(1, new KeyValuePair<string, object?>[]
+        {
+            new("Map", Map?.Name),
+            new("Region", Region?.Name),
+            new("IsPlayer", Player),
+        });
+
         Region.OnDeath(this);
         OnDeath(c);
     }

Projects/Server/Network/NetState/NetState.cs

@@ -23,6 +23,7 @@ using System;
 using System.Buffers;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.IO;
 using System.Net;
 using System.Net.Sockets;
@@ -37,6 +38,8 @@ public delegate int EncodePacket(ReadOnlySpan<byte> inputBuffer, Span<byte> outp
 
 public partial class NetState : IComparable<NetState>, IValueLinkListNode<NetState>
 {
+    private static readonly Counter<long> packetCounter =
+        Telemetry.NetworkMeter.CreateCounter<long>("net_state_packet_counter");
     private static readonly ILogger logger = LogFactory.GetLogger(typeof(NetState));
 
     private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds
@@ -476,6 +479,12 @@ public partial class NetState : IComparable<NetState>, IValueLinkListNode<NetSta
                 LogPacket(span, false);
             }
 
+            packetCounter.Add(1, new KeyValuePair<string, object>[]
+            {
+                new("PacketID", span[0]),
+                new("Kind", "SERVER"),
+            });
+
             SendPipe.Writer.Advance((uint)length);
 
             if (!_flushQueued)
@@ -564,6 +573,11 @@ public partial class NetState : IComparable<NetState>, IValueLinkListNode<NetSta
                 var packetId = packetReader.ReadByte();
                 int packetLength = length;
 
+                packetCounter.Add(1, new KeyValuePair<string, object>[]
+                {
+                    new("PacketID", packetId),
+                    new("Kind", "CLIENT"),
+                });
                 // These can arrive at any time and are only informational
                 if (_protocolState != ProtocolState.AwaitingSeed && IncomingPackets.IsInfoPacket(packetId))
                 {

Projects/Server/Network/TcpServer.cs

@@ -14,6 +14,7 @@
  *************************************************************************/
 
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.IO;
 using System.Linq;
 using System.Net;
@@ -27,6 +28,18 @@ namespace Server.Network;
 
 public static class TcpServer
 {
+    private static readonly Counter<long> attemptedConnectionsCounter =
+        Telemetry.NetworkMeter.CreateCounter<long>("tcp_server_attempted_connections_count");
+
+    private static readonly Counter<long> ipLimitedConnectionsCounter =
+        Telemetry.NetworkMeter.CreateCounter<long>("tcp_server_ip_limited_connections_count");
+
+    private static readonly Counter<long> firewalledConnectionsCounter =
+        Telemetry.NetworkMeter.CreateCounter<long>("tcp_server_firewalled_connections_count");
+
+    private static readonly Counter<long> allowedConnectionsCounter =
+        Telemetry.NetworkMeter.CreateCounter<long>("tcp_server_allowed_connections_count");
+
     private static readonly ILogger logger = LogFactory.GetLogger(typeof(TcpServer));
 
     // AccountLoginReject BadComm
@@ -78,11 +91,14 @@ public static class TcpServer
     }
 
     public static IEnumerable<IPEndPoint> GetListeningAddresses(IPEndPoint ipep) =>
-        NetworkInterface.GetAllNetworkInterfaces().SelectMany(adapter =>
+        NetworkInterface.GetAllNetworkInterfaces()
-            adapter.GetIPProperties().UnicastAddresses
+            .SelectMany(
-                .Where(uip => ipep.AddressFamily == uip.Address.AddressFamily)
+                adapter =>
-                .Select(uip => new IPEndPoint(uip.Address, ipep.Port))
+                    adapter.GetIPProperties()
-        );
+                        .UnicastAddresses
+                        .Where(uip => ipep.AddressFamily == uip.Address.AddressFamily)
+                        .Select(uip => new IPEndPoint(uip.Address, ipep.Port))
+            );
 
     public static Socket CreateListener(IPEndPoint ipep)
     {
@@ -133,13 +149,17 @@ public static class TcpServer
                 socket = await listener.AcceptAsync();
                 var remoteIP = ((IPEndPoint)socket.RemoteEndPoint)!.Address;
 
+                attemptedConnectionsCounter.Add(1);
+
                 if (!IPLimiter.Verify(remoteIP))
                 {
+                    ipLimitedConnectionsCounter.Add(1);
                     TraceDisconnect("Past IP limit threshold", remoteIP);
                     logger.Debug("{Address} Past IP limit threshold", remoteIP);
                 }
                 else if (Firewall.IsBlocked(remoteIP))
                 {
+                    firewalledConnectionsCounter.Add(1);
                     TraceDisconnect("Firewalled", remoteIP);
                     logger.Debug("{Address} Firewalled", remoteIP);
                 }
@@ -150,6 +170,7 @@ public static class TcpServer
 
                     if (args.AllowConnection)
                     {
+                        allowedConnectionsCounter.Add(1);
                         _ = new NetState(socket);
                         continue;
                     }

Projects/Server/Server.csproj

@@ -35,6 +35,9 @@
     <ItemGroup>
         <PackageReference Include="CommunityToolkit.HighPerformance" Version="8.3.2" />
         <PackageReference Include="LibDeflate.Bindings" Version="1.0.2.120" />
+        <PackageReference Include="OpenTelemetry" Version="1.10.0-beta.1" />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.10.0-beta.1" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
         <PackageReference Include="PollGroup" Version="1.5.1" />
         <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
 

Projects/Server/Telemetry.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Diagnostics.Metrics;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+
+namespace Server;
+
+public class Telemetry
+{
+    public static MeterProvider MeterProvider { get; private set; }
+    public static readonly Meter AccountingMeter = new("ModernUO.Accounting");
+    public static readonly Meter ItemsMeter = new("ModernUO.Items");
+    public static readonly Meter MobilesMeter = new("ModernUO.Mobiles");
+    public static readonly Meter NetworkMeter = new("ModernUO.Network");
+    public static readonly Meter EconomyMeter = new("ModernUO.Economy");
+
+    public static void Start()
+    {
+        MeterProvider = Sdk.CreateMeterProviderBuilder()
+            .ConfigureResource(r => r.AddService("ModernUO"))
+            .AddRuntimeInstrumentation()
+            .AddOtlpExporter(options =>
+            {
+                options.BatchExportProcessorOptions.ScheduledDelayMilliseconds = TimeSpan.FromSeconds(20.0).Milliseconds;
+            })
+            .AddMeter("ModernUO.Accounting")
+            .AddMeter("ModernUO.Items")
+            .AddMeter("ModernUO.Mobiles")
+            .AddMeter("ModernUO.Economy")
+            .AddMeter("ModernUO.Network")
+            .AddView(
+                "gold_created_histogram",
+                new ExplicitBucketHistogramConfiguration()
+                    { Boundaries = new double[] { 100, 200, 300, 400, 500, 700, 1000, 1500, 2000 } }
+            )
+            .Build();
+    }
+}

Projects/UOContent/Accounting/Account.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Net;
 using System.Runtime.CompilerServices;
 using System.Xml;
@@ -15,6 +16,9 @@ namespace Server.Accounting;
 [SerializationGenerator(6)]
 public partial class Account : IAccount, IComparable<Account>
 {
+    private static readonly Counter<long> accountsCreatedCounter = Telemetry.AccountingMeter.CreateCounter<long>("accounts_created_count");
+    private static readonly Counter<long> loginsCounter = Telemetry.AccountingMeter.CreateCounter<long>("logins_count");
+
     public static readonly TimeSpan YoungDuration = TimeSpan.FromHours(40.0);
     public static readonly TimeSpan InactiveDuration = TimeSpan.FromDays(180.0);
     public static readonly TimeSpan EmptyInactiveDuration = TimeSpan.FromDays(30.0);
@@ -133,6 +137,8 @@ public partial class Account : IAccount, IComparable<Account>
 
         Accounts.Add(this);
         this.MarkDirty();
+
+        accountsCreatedCounter.Add(1);
     }
 
     public Account(XmlElement node)
@@ -794,6 +800,8 @@ public partial class Account : IAccount, IComparable<Account>
                 m.SendAsciiMessage($"You will enjoy the benefits and relatively safe status of a young player for {hours} more hours.");
             }
         }
+
+        loginsCounter.Add(1, new KeyValuePair<string, object>("AccessLevel", $"{pm.AccessLevel}"));
     }
 
     public void RemoveYoungStatus(int message)

Projects/UOContent/Engines/Harvest/Core/HarvestSystem.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using Server.Items;
 using Server.Targeting;
 
@@ -6,6 +8,8 @@ namespace Server.Engines.Harvest
 {
     public abstract class HarvestSystem
     {
+        private static readonly Counter<long> harvestedResourcesCounter =
+            Telemetry.EconomyMeter.CreateCounter<long>("harvested_resources_count");
         public HarvestDefinition[] Definitions { get; init; }
 
         public virtual bool CheckTool(Mobile from, Item tool)
@@ -196,6 +200,13 @@ namespace Server.Engines.Harvest
 
                         bank.Consume(item.Amount, from);
 
+                        harvestedResourcesCounter.Add(item.Amount, new KeyValuePair<string, object>[]
+                        {
+                            new("Resource", item.GetType().Name),
+                            new("Map", map?.Name),
+                            new("Region", from?.Region.Name),
+                        });
+
                         if (Give(from, item, def.PlaceAtFeetIfFull))
                         {
                             SendSuccessTo(from, item, resource);

Projects/UOContent/Engines/Player Murder System/PlayerMurderSystem.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Diagnostics.Metrics;
 using System.Runtime.InteropServices;
 using Server.Collections;
+using Server.Ethics;
 using Server.Logging;
 using Server.Mobiles;
 
@@ -10,6 +12,7 @@ namespace Server.Engines.PlayerMurderSystem;
 
 public class PlayerMurderSystem : GenericPersistence
 {
+    private static readonly Counter<long> playerMurderCounter = Telemetry.MobilesMeter.CreateCounter<long>("player_murder_count");
     private static PlayerMurderSystem _playerMurderPersistence;
 
     private static readonly ILogger logger = LogFactory.GetLogger(typeof(PlayerMurderSystem));
@@ -166,6 +169,12 @@ public class PlayerMurderSystem : GenericPersistence
 
     public static void OnPlayerMurder(PlayerMobile player)
     {
+        playerMurderCounter.Add(1, new KeyValuePair<string, object>[]
+        {
+            new("Map", player.Map?.Name),
+            new("Region", player.Region?.Name),
+        });
+
         var context = GetOrCreateMurderContext(player);
         context.ShortTermMurders++;
         player.Kills++;

Projects/UOContent/Engines/Spawners/BaseSpawner.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Reflection;
 using System.Text.Json;
 using ModernUO.Serialization;
@@ -15,6 +16,9 @@ namespace Server.Engines.Spawners;
 [SerializationGenerator(10, false)]
 public abstract partial class BaseSpawner : Item, ISpawner
 {
+    private static readonly Counter<long> mobilesSpawnedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_spawned_count");
+    private static readonly Counter<long> itemsSpawnedCounter = Telemetry.ItemsMeter.CreateCounter<long>("items_spawned_count");
+
     [SerializedIgnoreDupe]
     [SerializableField(0)]
     [SerializedCommandProperty(AccessLevel.Developer)]
@@ -703,6 +707,12 @@ public abstract partial class BaseSpawner : Item, ISpawner
 
                 m.Spawner = this;
                 m.OnAfterSpawn();
+
+                mobilesSpawnedCounter.Add(1, new KeyValuePair<string, object?>[]
+                {
+                    new("Map", map),
+                    new("Region", m.Region),
+                });
             }
             else if (entity is Item item)
             {
@@ -717,6 +727,8 @@ public abstract partial class BaseSpawner : Item, ISpawner
 
                 item.Spawner = this;
                 item.OnAfterSpawn();
+
+                itemsSpawnedCounter.Add(1, new KeyValuePair<string, object?>("Map", map));
             }
             else
             {

Projects/UOContent/Items/Misc/Gold.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics.Metrics;
 using ModernUO.Serialization;
 using Server.Accounting;
 
@@ -7,6 +8,10 @@ namespace Server.Items;
 [SerializationGenerator(0, false)]
 public partial class Gold : Item
 {
+    private static readonly Counter<long> goldCreatedCounter = Telemetry.EconomyMeter.CreateCounter<long>("gold_created_count");
+    private static readonly Histogram<long> goldCreatedHistogram = Telemetry.EconomyMeter.CreateHistogram<long>("gold_created_histogram");
+    private static readonly Counter<long> goldDeletedCounter = Telemetry.EconomyMeter.CreateCounter<long>("gold_deleted_count");
+
     [Constructible]
     public Gold(int amountFrom, int amountTo) : this(Utility.RandomMinMax(amountFrom, amountTo))
     {
@@ -36,6 +41,19 @@ public partial class Gold : Item
         var newValue = Amount;
 
         UpdateTotal(this, TotalType.Gold, newValue - oldValue);
+
+        if (newValue > 1)
+        {
+            goldCreatedCounter.Add(newValue);
+            goldCreatedHistogram.Record(newValue);
+        }
+    }
+
+    public override void Delete()
+    {
+        goldDeletedCounter.Add(Amount);
+
+        base.Delete();
     }
 
     public override void OnAdded(IEntity parent)

Projects/UOContent/Mobiles/PlayerMobile.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using ModernUO.CodeGeneratedEvents;
 using Server.Accounting;
 using Server.Collections;
@@ -99,6 +100,7 @@ namespace Server.Mobiles
 
     public partial class PlayerMobile : Mobile, IHonorTarget, IHasSteps
     {
+        public static readonly Counter<long> playerDeathCounter = Telemetry.MobilesMeter.CreateCounter<long>("player_death_count");
         private static bool m_NoRecursion;
 
         private static readonly Point3D[] m_TrammelDeathDestinations =
@@ -2542,6 +2544,12 @@ namespace Server.Mobiles
 
             base.OnDeath(c);
 
+            playerDeathCounter.Add(1, new KeyValuePair<string, object>[]
+            {
+                new("Map", Map?.Name),
+                new("Region", Region?.Name),
+            });
+
             EquipSnapshot = null;
 
             HueMod = -1;

Projects/UOContent/Spells/Base/Spell.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using Server.Engines.ConPVP;
 using Server.Items;
 using Server.Misc;
@@ -15,6 +16,8 @@ namespace Server.Spells
 {
     public abstract class Spell : ISpell
     {
+        protected static readonly Counter<long> spellCastCount = Telemetry.MobilesMeter.CreateCounter<long>("spell_cast_count");
+
         private static readonly TimeSpan NextSpellDelay = TimeSpan.FromSeconds(0.75);
         private static readonly TimeSpan AnimateDelay = TimeSpan.FromSeconds(1.5);
         // In reality, it's ANY delayed Damage spell Post-AoS that can't stack, but, only
@@ -922,43 +925,52 @@ namespace Server.Spells
 
         private class CastTimer : Timer
         {
-            private readonly Spell m_Spell;
+            private readonly Spell _spell;
 
             public CastTimer(Spell spell, TimeSpan castDelay) : base(castDelay)
             {
-                m_Spell = spell;
+                _spell = spell;
             }
 
             protected override void OnTick()
             {
-                var caster = m_Spell?.Caster;
+                var caster = _spell?.Caster;
 
                 if (caster == null)
                 {
                     return;
                 }
 
-                if (m_Spell.State == SpellState.Casting && caster.Spell == m_Spell)
+                if (_spell.State == SpellState.Casting && caster.Spell == _spell)
                 {
-                    m_Spell.State = SpellState.Sequencing;
+                    _spell.State = SpellState.Sequencing;
-                    m_Spell._castTimer = null;
+
-                    caster.OnSpellCast(m_Spell);
+                    spellCastCount.Add(1, new KeyValuePair<string, object>[]
-                    caster.Region?.OnSpellCast(caster, m_Spell);
+                    {
+                        new("SpellName", _spell.Name),
+                        new("Skill", $"{_spell.CastSkill}"),
+                        new("Map", caster.Map?.Name),
+                        new("Region", caster.Region?.Name),
+                    });
+
+                    _spell._castTimer = null;
+                    caster.OnSpellCast(_spell);
+                    caster.Region?.OnSpellCast(caster, _spell);
                     caster.NextSpellTime =
-                        Core.TickCount + (int)m_Spell.GetCastRecovery().TotalMilliseconds; // Spell.NextSpellDelay;
+                        Core.TickCount + (int)_spell.GetCastRecovery().TotalMilliseconds; // Spell.NextSpellDelay;
 
                     caster.Delta(MobileDelta.Flags); // Update paralyze
 
                     var originalTarget = caster.Target;
 
-                    m_Spell.OnCast();
+                    _spell.OnCast();
 
                     if (caster.Player && caster.Target != originalTarget)
                     {
                         caster.Target?.BeginTimeout(caster, 30000); // 30 seconds
                     }
 
-                    m_Spell._castTimer = null;
+                    _spell._castTimer = null;
                 }
             }
 

Projects/UOContent/UOContent.csproj

@@ -44,6 +44,7 @@
         <PackageReference Include="Argon2.Bindings" Version="1.16.1" />
         <PackageReference Include="ModernUO.CodeGeneratedEvents.Annotations" Version="1.0.0" />
         <PackageReference Include="ModernUO.CodeGeneratedEvents.Generator" Version="1.0.3.2" PrivateAssets="all" />
+        <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="9.0.0-rc.2.24473.5" />
         <PackageReference Include="Zstd.Binaries" Version="1.6.0" />
 
         <PackageReference Include="ModernUO.Serialization.Annotations" Version="2.9.1" />

docker-compose.yml

@@ -0,0 +1,33 @@
+version: "2"
+services:
+  # Collector
+  otel-collector:
+    image: otel/opentelemetry-collector:0.88.0
+    restart: always
+    command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "8888:8888"   # Prometheus metrics exposed by the collector
+      - "8889:8889"   # Prometheus exporter metrics
+      - "13133:13133" # health_check extension
+      - "4317:4317"   # OTLP gRPC receiver
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    restart: always
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./grafana:/etc/grafana/provisioning/datasources
+      - ./grafana-storage:/var/lib/grafana

grafana/datasources.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    access: proxy
+    editable: true

otel-collector-config.yaml

@@ -0,0 +1,25 @@
+receivers:
+    otlp:
+        protocols:
+            grpc:
+                endpoint: 0.0.0.0:4317
+
+exporters:
+    prometheus:
+        endpoint: "0.0.0.0:8889"
+
+    debug:
+
+processors:
+    batch:
+
+extensions:
+    health_check:
+
+service:
+    extensions: [health_check]
+    pipelines:
+       metrics:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [debug, prometheus]

prometheus.yaml

@@ -0,0 +1,6 @@
+scrape_configs:
+    - job_name: 'otel-collector'
+      scrape_interval: 10s
+      static_configs:
+          - targets: ['otel-collector:8889']
+          - targets: ['otel-collector:8888']