OTEL POC

.gitignore

@@ -42,3 +42,5 @@
 
 /packages/*
 /Distribution/Configuration/server-access.json
+
+/grafana-storage/*

Projects/Server/Main.cs

@@ -417,6 +417,8 @@ public static class Core
 
         VerifySerialization();
 
+        Telemetry.Start();
+
         _now = DateTime.UtcNow;
         _firstTick = _tickCount = GetTimestamp();
 

Projects/Server/Mobiles/Mobile.cs

@@ -28,6 +28,7 @@ using Server.Targeting;
 using Server.Text;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Runtime.CompilerServices;
 using CalcMoves = Server.Movement.Movement;
 
@@ -196,6 +197,10 @@ public delegate int AOSStatusHandler(Mobile from, int index);
 /// </summary>
 public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPropertyListEntity, IValueLinkListNode<Mobile>
 {
+    private static readonly Counter<long> mobilesCreatedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_created_count");
+    private static readonly Counter<long> mobilesDeletedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_deleted_count");
+    private static readonly Counter<long> mobilesKilledCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_killed_count");
+
     // Allow four warmode changes in 0.5 seconds, any more will be delay for two seconds
     private const int WarmodeCatchCount = 4;
 
@@ -364,6 +369,11 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
         DefaultMobileInit();
 
         World.AddEntity(this);
+
+        mobilesCreatedCounter.Add(1, new KeyValuePair<string, object?>[]
+        {
+            new("Map", Map?.Name),
+        });
     }
 
     public Mobile(Serial serial)
@@ -2421,6 +2431,14 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
 
         Region.OnRegionChange(this, m_Region, null);
 
+        if (Alive) {
+            mobilesDeletedCounter.Add(1, new KeyValuePair<string, object?>[]
+            {
+                new("Map", Map?.Name),
+                new("Region", Region?.Name),
+            });
+        }
+
         m_Region = null;
         // Is the above line REALLY needed?  The old Region system did NOT have said line
         // and worked fine, because of this a LOT of extra checks have to be done everywhere...
@@ -4801,6 +4819,13 @@ public partial class Mobile : IHued, IComparable<Mobile>, ISpawnable, IObjectPro
             }
         }
 
+        mobilesKilledCounter.Add(1, new KeyValuePair<string, object?>[]
+        {
+            new("Map", Map?.Name),
+            new("Region", Region?.Name),
+            new("IsPlayer", Player),
+        });
+
         Region.OnDeath(this);
         OnDeath(c);
     }

Projects/Server/Server.csproj

@@ -35,6 +35,8 @@
     <ItemGroup>
         <PackageReference Include="CommunityToolkit.HighPerformance" Version="8.3.2" />
         <PackageReference Include="LibDeflate.Bindings" Version="1.0.2.120" />
+        <PackageReference Include="OpenTelemetry" Version="1.10.0-beta.1" />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.10.0-beta.1" />
         <PackageReference Include="PollGroup" Version="1.5.1" />
         <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
 

Projects/Server/Telemetry.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Diagnostics.Metrics;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+
+namespace Server;
+
+public class Telemetry
+{
+    public static MeterProvider MeterProvider { get; private set; }
+    public static readonly Meter AccountingMeter = new("ModernUO.Accounting");
+    public static readonly Meter ItemsMeter = new("ModernUO.Items");
+    public static readonly Meter MobilesMeter = new("ModernUO.Mobiles");
+    public static readonly Meter EconomyMeter = new("ModernUO.Economy");
+
+    public static void Start()
+    {
+        MeterProvider = Sdk.CreateMeterProviderBuilder()
+            .ConfigureResource(r => r.AddService("ModernUO"))
+            .AddOtlpExporter(options =>
+            {
+                options.BatchExportProcessorOptions.ScheduledDelayMilliseconds = TimeSpan.FromSeconds(10.0).Milliseconds;
+            })
+            .AddMeter("ModernUO.Accounting")
+            .AddMeter("ModernUO.Items")
+            .AddMeter("ModernUO.Mobiles")
+            .AddMeter("ModernUO.Economy")
+            .AddView(
+                "gold_created_histogram",
+                new ExplicitBucketHistogramConfiguration()
+                    { Boundaries = new double[] { 100, 200, 300, 400, 500, 700, 1000, 1500, 2000 } }
+            )
+            .Build();
+    }
+}

Projects/UOContent/Accounting/Account.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Net;
 using System.Runtime.CompilerServices;
 using System.Xml;
@@ -15,6 +16,9 @@ namespace Server.Accounting;
 [SerializationGenerator(6)]
 public partial class Account : IAccount, IComparable<Account>
 {
+    private static readonly Counter<long> accountsCreatedCounter = Telemetry.AccountingMeter.CreateCounter<long>("accounts_created_count");
+    private static readonly Counter<long> loginsCounter = Telemetry.AccountingMeter.CreateCounter<long>("logins_count");
+
     public static readonly TimeSpan YoungDuration = TimeSpan.FromHours(40.0);
     public static readonly TimeSpan InactiveDuration = TimeSpan.FromDays(180.0);
     public static readonly TimeSpan EmptyInactiveDuration = TimeSpan.FromDays(30.0);
@@ -133,6 +137,8 @@ public partial class Account : IAccount, IComparable<Account>
 
         Accounts.Add(this);
         this.MarkDirty();
+
+        accountsCreatedCounter.Add(1);
     }
 
     public Account(XmlElement node)
@@ -794,6 +800,8 @@ public partial class Account : IAccount, IComparable<Account>
                 m.SendAsciiMessage($"You will enjoy the benefits and relatively safe status of a young player for {hours} more hours.");
             }
         }
+
+        loginsCounter.Add(1, new KeyValuePair<string, object>("AccessLevel", $"{pm.AccessLevel}"));
     }
 
     public void RemoveYoungStatus(int message)

Projects/UOContent/Engines/Spawners/BaseSpawner.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Reflection;
 using System.Text.Json;
 using ModernUO.Serialization;
@@ -15,6 +16,9 @@ namespace Server.Engines.Spawners;
 [SerializationGenerator(10, false)]
 public abstract partial class BaseSpawner : Item, ISpawner
 {
+    private static readonly Counter<long> mobilesSpawnedCounter = Telemetry.MobilesMeter.CreateCounter<long>("mobiles_spawned_count");
+    private static readonly Counter<long> itemsSpawnedCounter = Telemetry.ItemsMeter.CreateCounter<long>("items_spawned_count");
+
     [SerializedIgnoreDupe]
     [SerializableField(0)]
     [SerializedCommandProperty(AccessLevel.Developer)]
@@ -703,6 +707,12 @@ public abstract partial class BaseSpawner : Item, ISpawner
 
                 m.Spawner = this;
                 m.OnAfterSpawn();
+
+                mobilesSpawnedCounter.Add(1, new KeyValuePair<string, object?>[]
+                {
+                    new("Map", map),
+                    new("Region", m.Region),
+                });
             }
             else if (entity is Item item)
             {
@@ -717,6 +727,8 @@ public abstract partial class BaseSpawner : Item, ISpawner
 
                 item.Spawner = this;
                 item.OnAfterSpawn();
+
+                itemsSpawnedCounter.Add(1, new KeyValuePair<string, object?>("Map", map));
             }
             else
             {

Projects/UOContent/Items/Misc/Gold.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics.Metrics;
 using ModernUO.Serialization;
 using Server.Accounting;
 
@@ -7,6 +8,10 @@ namespace Server.Items;
 [SerializationGenerator(0, false)]
 public partial class Gold : Item
 {
+    private static readonly Counter<long> goldCreatedCounter = Telemetry.EconomyMeter.CreateCounter<long>("gold_created_count");
+    private static readonly Histogram<long> goldCreatedHistogram = Telemetry.EconomyMeter.CreateHistogram<long>("gold_created_histogram");
+    private static readonly Counter<long> goldDeletedCounter = Telemetry.EconomyMeter.CreateCounter<long>("gold_deleted_count");
+
     [Constructible]
     public Gold(int amountFrom, int amountTo) : this(Utility.RandomMinMax(amountFrom, amountTo))
     {
@@ -36,6 +41,19 @@ public partial class Gold : Item
         var newValue = Amount;
 
         UpdateTotal(this, TotalType.Gold, newValue - oldValue);
+
+        if (newValue > 1)
+        {
+            goldCreatedCounter.Add(newValue);
+            goldCreatedHistogram.Record(newValue);
+        }
+    }
+
+    public override void Delete()
+    {
+        goldDeletedCounter.Add(Amount);
+
+        base.Delete();
     }
 
     public override void OnAdded(IEntity parent)

Projects/UOContent/UOContent.csproj

@@ -44,6 +44,7 @@
         <PackageReference Include="Argon2.Bindings" Version="1.16.1" />
         <PackageReference Include="ModernUO.CodeGeneratedEvents.Annotations" Version="1.0.0" />
         <PackageReference Include="ModernUO.CodeGeneratedEvents.Generator" Version="1.0.3.2" PrivateAssets="all" />
+        <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="9.0.0-rc.2.24473.5" />
         <PackageReference Include="Zstd.Binaries" Version="1.6.0" />
 
         <PackageReference Include="ModernUO.Serialization.Annotations" Version="2.9.1" />

docker-compose.yml

@@ -0,0 +1,33 @@
+version: "2"
+services:
+  # Collector
+  otel-collector:
+    image: otel/opentelemetry-collector:0.88.0
+    restart: always
+    command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "8888:8888"   # Prometheus metrics exposed by the collector
+      - "8889:8889"   # Prometheus exporter metrics
+      - "13133:13133" # health_check extension
+      - "4317:4317"   # OTLP gRPC receiver
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    restart: always
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./grafana:/etc/grafana/provisioning/datasources
+      - ./grafana-storage:/var/lib/grafana

grafana/datasources.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    access: proxy
+    editable: true

otel-collector-config.yaml

@@ -0,0 +1,25 @@
+receivers:
+    otlp:
+        protocols:
+            grpc:
+                endpoint: 0.0.0.0:4317
+
+exporters:
+    prometheus:
+        endpoint: "0.0.0.0:8889"
+
+    debug:
+
+processors:
+    batch:
+
+extensions:
+    health_check:
+
+service:
+    extensions: [health_check]
+    pipelines:
+       metrics:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [debug, prometheus]

prometheus.yaml

@@ -0,0 +1,6 @@
+scrape_configs:
+    - job_name: 'otel-collector'
+      scrape_interval: 10s
+      static_configs:
+          - targets: ['otel-collector:8889']
+          - targets: ['otel-collector:8888']