diff --git a/.gitignore b/.gitignore index a0bf66daf..5714d07be 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ /packages/* /Distribution/Configuration/server-access.json + +/grafana-storage/* diff --git a/Projects/Server/Main.cs b/Projects/Server/Main.cs index 535327cbe..38cb0db48 100644 --- a/Projects/Server/Main.cs +++ b/Projects/Server/Main.cs @@ -420,6 +420,8 @@ public static class Core VerifySerialization(); + Telemetry.Start(); + _now = DateTime.UtcNow; _firstTick = _tickCount = GetTimestamp(); diff --git a/Projects/Server/Mobiles/Mobile.cs b/Projects/Server/Mobiles/Mobile.cs index d6b9eb9a2..b86353f1b 100644 --- a/Projects/Server/Mobiles/Mobile.cs +++ b/Projects/Server/Mobiles/Mobile.cs @@ -28,6 +28,7 @@ using Server.Targeting; using Server.Text; using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.Runtime.CompilerServices; using CalcMoves = Server.Movement.Movement; @@ -196,6 +197,10 @@ public delegate int AOSStatusHandler(Mobile from, int index); /// public partial class Mobile : IHued, IComparable, ISpawnable, IObjectPropertyListEntity, IValueLinkListNode { + private static readonly Counter mobilesCreatedCounter = Telemetry.MobilesMeter.CreateCounter("mobiles_created_count"); + private static readonly Counter mobilesDeletedCounter = Telemetry.MobilesMeter.CreateCounter("mobiles_deleted_count"); + private static readonly Counter mobilesKilledCounter = Telemetry.MobilesMeter.CreateCounter("mobiles_killed_count"); + // Allow four warmode changes in 0.5 seconds, any more will be delay for two seconds private const int WarmodeCatchCount = 4; @@ -364,6 +369,11 @@ public partial class Mobile : IHued, IComparable, ISpawnable, IObjectPro DefaultMobileInit(); World.AddEntity(this); + + mobilesCreatedCounter.Add(1, new KeyValuePair[] + { + new("Map", Map?.Name), + }); } public Mobile(Serial serial) @@ -2419,6 +2429,14 @@ public partial class Mobile : IHued, IComparable, ISpawnable, IObjectPro DropHolding(); + if (Alive) { + mobilesDeletedCounter.Add(1, new KeyValuePair[] + { + new("Map", Map?.Name), + new("Region", Region?.Name), + }); + } + Region.OnRegionChange(this, m_Region, null); m_Region = null; @@ -4801,6 +4819,13 @@ public partial class Mobile : IHued, IComparable, ISpawnable, IObjectPro } } + mobilesKilledCounter.Add(1, new KeyValuePair[] + { + new("Map", Map?.Name), + new("Region", Region?.Name), + new("IsPlayer", Player), + }); + Region.OnDeath(this); OnDeath(c); } diff --git a/Projects/Server/Network/NetState/NetState.cs b/Projects/Server/Network/NetState/NetState.cs index 172be42aa..18f94743c 100755 --- a/Projects/Server/Network/NetState/NetState.cs +++ b/Projects/Server/Network/NetState/NetState.cs @@ -23,6 +23,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.IO; using System.Net; using System.Net.Sockets; @@ -37,6 +38,8 @@ public delegate int EncodePacket(ReadOnlySpan inputBuffer, Span outp public partial class NetState : IComparable, IValueLinkListNode { + private static readonly Counter packetCounter = + Telemetry.NetworkMeter.CreateCounter("net_state_packet_counter"); private static readonly ILogger logger = LogFactory.GetLogger(typeof(NetState)); private static readonly TimeSpan ConnectingSocketIdleLimit = TimeSpan.FromMilliseconds(5000); // 5 seconds @@ -476,6 +479,12 @@ public partial class NetState : IComparable, IValueLinkListNode[] + { + new("PacketID", span[0]), + new("Kind", "SERVER"), + }); + SendPipe.Writer.Advance((uint)length); if (!_flushQueued) @@ -564,6 +573,11 @@ public partial class NetState : IComparable, IValueLinkListNode[] + { + new("PacketID", packetId), + new("Kind", "CLIENT"), + }); // These can arrive at any time and are only informational if (_protocolState != ProtocolState.AwaitingSeed && IncomingPackets.IsInfoPacket(packetId)) { diff --git a/Projects/Server/Network/TcpServer.cs b/Projects/Server/Network/TcpServer.cs index be526f703..8a77dcba6 100644 --- a/Projects/Server/Network/TcpServer.cs +++ b/Projects/Server/Network/TcpServer.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.IO; using System.Linq; using System.Net; @@ -27,6 +28,18 @@ namespace Server.Network; public static class TcpServer { + private static readonly Counter attemptedConnectionsCounter = + Telemetry.NetworkMeter.CreateCounter("tcp_server_attempted_connections_count"); + + private static readonly Counter ipLimitedConnectionsCounter = + Telemetry.NetworkMeter.CreateCounter("tcp_server_ip_limited_connections_count"); + + private static readonly Counter firewalledConnectionsCounter = + Telemetry.NetworkMeter.CreateCounter("tcp_server_firewalled_connections_count"); + + private static readonly Counter allowedConnectionsCounter = + Telemetry.NetworkMeter.CreateCounter("tcp_server_allowed_connections_count"); + private static readonly ILogger logger = LogFactory.GetLogger(typeof(TcpServer)); // AccountLoginReject BadComm @@ -78,11 +91,14 @@ public static class TcpServer } public static IEnumerable GetListeningAddresses(IPEndPoint ipep) => - NetworkInterface.GetAllNetworkInterfaces().SelectMany(adapter => - adapter.GetIPProperties().UnicastAddresses - .Where(uip => ipep.AddressFamily == uip.Address.AddressFamily) - .Select(uip => new IPEndPoint(uip.Address, ipep.Port)) - ); + NetworkInterface.GetAllNetworkInterfaces() + .SelectMany( + adapter => + adapter.GetIPProperties() + .UnicastAddresses + .Where(uip => ipep.AddressFamily == uip.Address.AddressFamily) + .Select(uip => new IPEndPoint(uip.Address, ipep.Port)) + ); public static Socket CreateListener(IPEndPoint ipep) { @@ -133,13 +149,17 @@ public static class TcpServer socket = await listener.AcceptAsync(); var remoteIP = ((IPEndPoint)socket.RemoteEndPoint)!.Address; + attemptedConnectionsCounter.Add(1); + if (!IPLimiter.Verify(remoteIP)) { + ipLimitedConnectionsCounter.Add(1); TraceDisconnect("Past IP limit threshold", remoteIP); logger.Debug("{Address} Past IP limit threshold", remoteIP); } else if (Firewall.IsBlocked(remoteIP)) { + firewalledConnectionsCounter.Add(1); TraceDisconnect("Firewalled", remoteIP); logger.Debug("{Address} Firewalled", remoteIP); } @@ -150,6 +170,7 @@ public static class TcpServer if (args.AllowConnection) { + allowedConnectionsCounter.Add(1); _ = new NetState(socket); continue; } diff --git a/Projects/Server/Server.csproj b/Projects/Server/Server.csproj index a47b3fb1a..a8a9907c7 100644 --- a/Projects/Server/Server.csproj +++ b/Projects/Server/Server.csproj @@ -35,6 +35,9 @@ + + + diff --git a/Projects/Server/Telemetry.cs b/Projects/Server/Telemetry.cs new file mode 100644 index 000000000..75164b6a7 --- /dev/null +++ b/Projects/Server/Telemetry.cs @@ -0,0 +1,39 @@ +using System; +using System.Diagnostics.Metrics; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; + +namespace Server; + +public class Telemetry +{ + public static MeterProvider MeterProvider { get; private set; } + public static readonly Meter AccountingMeter = new("ModernUO.Accounting"); + public static readonly Meter ItemsMeter = new("ModernUO.Items"); + public static readonly Meter MobilesMeter = new("ModernUO.Mobiles"); + public static readonly Meter NetworkMeter = new("ModernUO.Network"); + public static readonly Meter EconomyMeter = new("ModernUO.Economy"); + + public static void Start() + { + MeterProvider = Sdk.CreateMeterProviderBuilder() + .ConfigureResource(r => r.AddService("ModernUO")) + .AddRuntimeInstrumentation() + .AddOtlpExporter(options => + { + options.BatchExportProcessorOptions.ScheduledDelayMilliseconds = TimeSpan.FromSeconds(20.0).Milliseconds; + }) + .AddMeter("ModernUO.Accounting") + .AddMeter("ModernUO.Items") + .AddMeter("ModernUO.Mobiles") + .AddMeter("ModernUO.Economy") + .AddMeter("ModernUO.Network") + .AddView( + "gold_created_histogram", + new ExplicitBucketHistogramConfiguration() + { Boundaries = new double[] { 100, 200, 300, 400, 500, 700, 1000, 1500, 2000 } } + ) + .Build(); + } +} diff --git a/Projects/UOContent/Accounting/Account.cs b/Projects/UOContent/Accounting/Account.cs index ac7da1abe..8626a1b63 100644 --- a/Projects/UOContent/Accounting/Account.cs +++ b/Projects/UOContent/Accounting/Account.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.Net; using System.Runtime.CompilerServices; using System.Xml; @@ -15,6 +16,9 @@ namespace Server.Accounting; [SerializationGenerator(6)] public partial class Account : IAccount, IComparable { + private static readonly Counter accountsCreatedCounter = Telemetry.AccountingMeter.CreateCounter("accounts_created_count"); + private static readonly Counter loginsCounter = Telemetry.AccountingMeter.CreateCounter("logins_count"); + public static readonly TimeSpan YoungDuration = TimeSpan.FromHours(40.0); public static readonly TimeSpan InactiveDuration = TimeSpan.FromDays(180.0); public static readonly TimeSpan EmptyInactiveDuration = TimeSpan.FromDays(30.0); @@ -133,6 +137,8 @@ public partial class Account : IAccount, IComparable Accounts.Add(this); this.MarkDirty(); + + accountsCreatedCounter.Add(1); } public Account(XmlElement node) @@ -794,6 +800,8 @@ public partial class Account : IAccount, IComparable m.SendAsciiMessage($"You will enjoy the benefits and relatively safe status of a young player for {hours} more hours."); } } + + loginsCounter.Add(1, new KeyValuePair("AccessLevel", $"{pm.AccessLevel}")); } public void RemoveYoungStatus(int message) diff --git a/Projects/UOContent/Engines/Harvest/Core/HarvestSystem.cs b/Projects/UOContent/Engines/Harvest/Core/HarvestSystem.cs index 84337d61d..8864324b5 100644 --- a/Projects/UOContent/Engines/Harvest/Core/HarvestSystem.cs +++ b/Projects/UOContent/Engines/Harvest/Core/HarvestSystem.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; using Server.Items; using Server.Targeting; @@ -6,6 +8,8 @@ namespace Server.Engines.Harvest { public abstract class HarvestSystem { + private static readonly Counter harvestedResourcesCounter = + Telemetry.EconomyMeter.CreateCounter("harvested_resources_count"); public HarvestDefinition[] Definitions { get; init; } public virtual bool CheckTool(Mobile from, Item tool) @@ -196,6 +200,13 @@ namespace Server.Engines.Harvest bank.Consume(item.Amount, from); + harvestedResourcesCounter.Add(item.Amount, new KeyValuePair[] + { + new("Resource", item.GetType().Name), + new("Map", map?.Name), + new("Region", from?.Region.Name), + }); + if (Give(from, item, def.PlaceAtFeetIfFull)) { SendSuccessTo(from, item, resource); diff --git a/Projects/UOContent/Engines/Player Murder System/PlayerMurderSystem.cs b/Projects/UOContent/Engines/Player Murder System/PlayerMurderSystem.cs index 7fda051ff..5785bb226 100644 --- a/Projects/UOContent/Engines/Player Murder System/PlayerMurderSystem.cs +++ b/Projects/UOContent/Engines/Player Murder System/PlayerMurderSystem.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Runtime.InteropServices; using Server.Collections; +using Server.Ethics; using Server.Logging; using Server.Mobiles; @@ -10,6 +12,7 @@ namespace Server.Engines.PlayerMurderSystem; public class PlayerMurderSystem : GenericPersistence { + private static readonly Counter playerMurderCounter = Telemetry.MobilesMeter.CreateCounter("player_murder_count"); private static PlayerMurderSystem _playerMurderPersistence; private static readonly ILogger logger = LogFactory.GetLogger(typeof(PlayerMurderSystem)); @@ -166,6 +169,12 @@ public class PlayerMurderSystem : GenericPersistence public static void OnPlayerMurder(PlayerMobile player) { + playerMurderCounter.Add(1, new KeyValuePair[] + { + new("Map", player.Map?.Name), + new("Region", player.Region?.Name), + }); + var context = GetOrCreateMurderContext(player); context.ShortTermMurders++; player.Kills++; diff --git a/Projects/UOContent/Engines/Spawners/BaseSpawner.cs b/Projects/UOContent/Engines/Spawners/BaseSpawner.cs index 28e423313..ee6e264d5 100644 --- a/Projects/UOContent/Engines/Spawners/BaseSpawner.cs +++ b/Projects/UOContent/Engines/Spawners/BaseSpawner.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.Reflection; using System.Text.Json; using ModernUO.Serialization; @@ -15,6 +16,9 @@ namespace Server.Engines.Spawners; [SerializationGenerator(10, false)] public abstract partial class BaseSpawner : Item, ISpawner { + private static readonly Counter mobilesSpawnedCounter = Telemetry.MobilesMeter.CreateCounter("mobiles_spawned_count"); + private static readonly Counter itemsSpawnedCounter = Telemetry.ItemsMeter.CreateCounter("items_spawned_count"); + [SerializedIgnoreDupe] [SerializableField(0)] [SerializedCommandProperty(AccessLevel.Developer)] @@ -703,6 +707,12 @@ public abstract partial class BaseSpawner : Item, ISpawner m.Spawner = this; m.OnAfterSpawn(); + + mobilesSpawnedCounter.Add(1, new KeyValuePair[] + { + new("Map", map), + new("Region", m.Region), + }); } else if (entity is Item item) { @@ -717,6 +727,8 @@ public abstract partial class BaseSpawner : Item, ISpawner item.Spawner = this; item.OnAfterSpawn(); + + itemsSpawnedCounter.Add(1, new KeyValuePair("Map", map)); } else { diff --git a/Projects/UOContent/Items/Misc/Gold.cs b/Projects/UOContent/Items/Misc/Gold.cs index af606187f..f5fb77598 100644 --- a/Projects/UOContent/Items/Misc/Gold.cs +++ b/Projects/UOContent/Items/Misc/Gold.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.Metrics; using ModernUO.Serialization; using Server.Accounting; @@ -7,6 +8,10 @@ namespace Server.Items; [SerializationGenerator(0, false)] public partial class Gold : Item { + private static readonly Counter goldCreatedCounter = Telemetry.EconomyMeter.CreateCounter("gold_created_count"); + private static readonly Histogram goldCreatedHistogram = Telemetry.EconomyMeter.CreateHistogram("gold_created_histogram"); + private static readonly Counter goldDeletedCounter = Telemetry.EconomyMeter.CreateCounter("gold_deleted_count"); + [Constructible] public Gold(int amountFrom, int amountTo) : this(Utility.RandomMinMax(amountFrom, amountTo)) { @@ -36,6 +41,19 @@ public partial class Gold : Item var newValue = Amount; UpdateTotal(this, TotalType.Gold, newValue - oldValue); + + if (newValue > 1) + { + goldCreatedCounter.Add(newValue); + goldCreatedHistogram.Record(newValue); + } + } + + public override void Delete() + { + goldDeletedCounter.Add(Amount); + + base.Delete(); } public override void OnAdded(IEntity parent) diff --git a/Projects/UOContent/Mobiles/PlayerMobile.cs b/Projects/UOContent/Mobiles/PlayerMobile.cs index f9b607cc9..5af206a61 100644 --- a/Projects/UOContent/Mobiles/PlayerMobile.cs +++ b/Projects/UOContent/Mobiles/PlayerMobile.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using ModernUO.CodeGeneratedEvents; using Server.Accounting; using Server.Collections; @@ -99,6 +100,7 @@ namespace Server.Mobiles public partial class PlayerMobile : Mobile, IHonorTarget, IHasSteps { + public static readonly Counter playerDeathCounter = Telemetry.MobilesMeter.CreateCounter("player_death_count"); private static bool m_NoRecursion; private static readonly Point3D[] m_TrammelDeathDestinations = @@ -2542,6 +2544,12 @@ namespace Server.Mobiles base.OnDeath(c); + playerDeathCounter.Add(1, new KeyValuePair[] + { + new("Map", Map?.Name), + new("Region", Region?.Name), + }); + EquipSnapshot = null; HueMod = -1; diff --git a/Projects/UOContent/Spells/Base/Spell.cs b/Projects/UOContent/Spells/Base/Spell.cs index ec04e9158..fa4dcfdce 100644 --- a/Projects/UOContent/Spells/Base/Spell.cs +++ b/Projects/UOContent/Spells/Base/Spell.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using Server.Engines.ConPVP; using Server.Items; using Server.Misc; @@ -15,6 +16,8 @@ namespace Server.Spells { public abstract class Spell : ISpell { + protected static readonly Counter spellCastCount = Telemetry.MobilesMeter.CreateCounter("spell_cast_count"); + private static readonly TimeSpan NextSpellDelay = TimeSpan.FromSeconds(0.75); private static readonly TimeSpan AnimateDelay = TimeSpan.FromSeconds(1.5); // In reality, it's ANY delayed Damage spell Post-AoS that can't stack, but, only @@ -922,43 +925,52 @@ namespace Server.Spells private class CastTimer : Timer { - private readonly Spell m_Spell; + private readonly Spell _spell; public CastTimer(Spell spell, TimeSpan castDelay) : base(castDelay) { - m_Spell = spell; + _spell = spell; } protected override void OnTick() { - var caster = m_Spell?.Caster; + var caster = _spell?.Caster; if (caster == null) { return; } - if (m_Spell.State == SpellState.Casting && caster.Spell == m_Spell) + if (_spell.State == SpellState.Casting && caster.Spell == _spell) { - m_Spell.State = SpellState.Sequencing; - m_Spell._castTimer = null; - caster.OnSpellCast(m_Spell); - caster.Region?.OnSpellCast(caster, m_Spell); + _spell.State = SpellState.Sequencing; + + spellCastCount.Add(1, new KeyValuePair[] + { + new("SpellName", _spell.Name), + new("Skill", $"{_spell.CastSkill}"), + new("Map", caster.Map?.Name), + new("Region", caster.Region?.Name), + }); + + _spell._castTimer = null; + caster.OnSpellCast(_spell); + caster.Region?.OnSpellCast(caster, _spell); caster.NextSpellTime = - Core.TickCount + (int)m_Spell.GetCastRecovery().TotalMilliseconds; // Spell.NextSpellDelay; + Core.TickCount + (int)_spell.GetCastRecovery().TotalMilliseconds; // Spell.NextSpellDelay; caster.Delta(MobileDelta.Flags); // Update paralyze var originalTarget = caster.Target; - m_Spell.OnCast(); + _spell.OnCast(); if (caster.Player && caster.Target != originalTarget) { caster.Target?.BeginTimeout(caster, 30000); // 30 seconds } - m_Spell._castTimer = null; + _spell._castTimer = null; } } diff --git a/Projects/UOContent/UOContent.csproj b/Projects/UOContent/UOContent.csproj index 5f07339eb..c45d74490 100644 --- a/Projects/UOContent/UOContent.csproj +++ b/Projects/UOContent/UOContent.csproj @@ -44,6 +44,7 @@ + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..96b02f849 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +version: "2" +services: + # Collector + otel-collector: + image: otel/opentelemetry-collector:0.88.0 + restart: always + command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: always + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + + grafana: + image: grafana/grafana + container_name: grafana + restart: unless-stopped + ports: + - '3000:3000' + volumes: + - ./grafana:/etc/grafana/provisioning/datasources + - ./grafana-storage:/var/lib/grafana diff --git a/grafana/datasources.yml b/grafana/datasources.yml new file mode 100644 index 000000000..44999d469 --- /dev/null +++ b/grafana/datasources.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true diff --git a/otel-collector-config.yaml b/otel-collector-config.yaml new file mode 100644 index 000000000..ef0b07a4d --- /dev/null +++ b/otel-collector-config.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + + debug: + +processors: + batch: + +extensions: + health_check: + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug, prometheus] diff --git a/prometheus.yaml b/prometheus.yaml new file mode 100644 index 000000000..0120ad33d --- /dev/null +++ b/prometheus.yaml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['otel-collector:8889'] + - targets: ['otel-collector:8888']